diff --git a/packages/coding-agent/src/experimental/vacation/vacation.ts b/packages/coding-agent/src/experimental/vacation/vacation.ts index d6fc87513..8dde4bb81 100644 --- a/packages/coding-agent/src/experimental/vacation/vacation.ts +++ b/packages/coding-agent/src/experimental/vacation/vacation.ts @@ -122,7 +122,7 @@ const research = defineTool({ "You are a research subagent for a vacation planner. Search weather, museums, and trains in parallel, in one step, then report the findings in a few short bullet points.", }); }, context); - return { content: [{ type: "text", text: "Research started in the background." }] }; + return { output: [{ type: "text", text: "Research started in the background." }] }; }, }); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index 06c10b716..52cd1f3b2 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -2,6 +2,28 @@ ## [Unreleased] +### Breaking Changes + +- `ToolTaskInput` and `ToolTaskResult` are tagged unions: a model-issued call is `{ kind: "model", ... }` with its previous fields, a nested call `{ kind: "nested", ... }`. `pi.tool` is version 2 and migrates live version 1 tasks. +- `ToolExecutionResult.content` is now `output`, next to the new `structuredOutput`. +- `bash` and `powershell` answer a nonzero exit with an error result carrying an `exit_code` diagnostic, instead of throwing (`tool_error`). +- Any error a `Storage` method throws, a read included, now fails the Session, which closes itself: every later call and every pending wait rejects with `SessionFailed`, whose `cause` is that error, and nothing is retried. `StorageRejected` is removed; a storage retries its own transient failures and throws the new `StorageRequestError` for an invalid request, which fails only that call. A throw in the scheduler's own commits, or in the commit listeners of the Harness's components, fails the Session too. +- `subscribeCommits()` and `subscribeClose()` listeners that throw, or return a rejected promise, are reported through `onReport` instead of failing the operation that ran them. + +### Added + +- `Session.closed` settles once the Session has closed, Storage included: `{ reason: "closed" }` after `close()`, or `{ reason: "failed", error }` after a storage error. `WatchEnd` gains `{ reason: "session_failed", error }`, and the README has an Errors section. +- `ToolExecutionApi.executeTool(name, args, context, { key?, progress? })`: a tool calls another tool as a nested call. The nested call is its own `pi.tool` task owned by the caller, runs validation, the `ToolTask` hooks (whose `ToolHookCall` carries `parent`), output limits, and its replay policy, and returns a `NestedToolExecutionResult` (its `taskId`, `structuredOutput`, `details`, `diagnostics`, `usage`, `isError`, `durationMs`, but not the model's `output`) to the caller instead of the transcript. The result is kept, as returned, in `NestedResultDoc`, a task-scoped document family of the caller, so it is gone from storage once the caller settles; its task's receipt stays small. Keys, by default the call's position, let a replay-safe caller's rerun reattach to its nested calls; a caller that is not replay-safe has its unfinished nested calls abandoned after a restart. Nested calls show in `pi.live.nestedTools`, with the arguments they run with and a `summary` (error, duration, usage) once done (`progress: false` keeps their running output out), and as tool events with `parentToolCallId` and `parentTaskId`; every tool event also carries its call's `taskId`. A caller aborts the nested calls it left running before it settles. +- Structured tool output: a tool declares `structuredOutputSchema` and returns `structuredOutput`, which the Harness validates and gives to programs that call the tool; a tool without a schema gives them its bounded output, one text item as a string, one image as itself, and otherwise the content list, for errors too. A nested call with invalid structured output gets an error result; a model-issued one only loses it, and the host gets a report. `api.retainedOutput()` reads the bounded output the model will see. `bash` and `powershell` return `{ output, truncated, fullOutputPath?, exitCode }`. +- `ToolRegistration.callers` (`model`, `tools`, or both, the default) and `pi.agent` `modelTools`: who may call a tool, and which of its tools a conversation offers the model; `Agent.callable` lists the tools nested calls resolve among. `addTools` also extends `modelTools`. +- `TaskOptions.abandonOnRestart`: a task its creator awaits only in memory is aborted, with its owned work, when a later Harness starts scheduling, before any of it runs again. Its mark carries `abortReason: "restart"`, which cascades; such a task waits for a missing definition instead of becoming `orphaned`. Nested calls and `api.createTask()` children of tools that are not replay-safe get it by default. No task below an owner with cancellation intent starts a run phase any more, even before the cascade marks it. +- `TaskRuntime.abortOwned(id, context)`: abort a task this task owns and wait until it is terminal. + +### Fixed + +- A throwing `onReport` no longer replaces the error being reported or becomes an unhandled rejection, a throwing `now()` is reported once and `Date.now` used, and a watch listener's error is reported as well as ending the watch with `listener_error`. +- A task whose fault write failed no longer runs again in the same process. + ## [1.1.0] - 2026-10-07 ### Breaking Changes diff --git a/packages/durable/README.md b/packages/durable/README.md index cc3dd7099..765fea34a 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -32,6 +32,7 @@ Built on [`@earendil-works/pi-ai`](../ai/README.md) for model access and `@earen - [Your Own State](#your-own-state) - [Usage and Cost](#usage-and-cost) - [Storage](#storage) +- [Errors](#errors) - [Examples](#examples) - [Design Documents](#design-documents) @@ -148,7 +149,7 @@ An extension may bring `tools`, `sections`, `hooks`, `wraps` (decorators of a to `@earendil-works/pi-durable/tools` provides `read`, `write`, `edit`, and `bash`, and the `CodingTools` extension with all four. They touch files and processes only through the call's environment (see [Environment](#environment)). Reading images is not supported yet. -Define your own tool with a TypeBox schema. `defineTool()` types `args` from `parameters`, which the Harness validates before `execute()`. `api.output()` streams running output, which becomes the result when `execute()` returns no `content`: +Define your own tool with a TypeBox schema. `defineTool()` types `args` from `parameters`, which the Harness validates before `execute()`. `api.output()` streams running output, which becomes the result when `execute()` returns no `output`: ```typescript import { Type } from "@earendil-works/pi-ai"; @@ -167,6 +168,24 @@ registry.install(defineExtension({ name: "count", tools: [count] })); Each call runs as its own durable task. Its intent is committed before `execute()` runs. If the process dies mid-call, the tool reruns on reopen only when it is declared `replay: "safe"`; otherwise the model gets an `interrupted` error result with the output committed so far. Throwing from `execute()` gives the model an error result. A result can also return `usage`, which is added to the conversation's [usage](#usage-and-cost). It can also return `control: { terminate: true }`: when every result of the round asks for it, the run ends without another model request. +A result has three channels: `output` for the model, `structuredOutput` for programs that call the tool (see [Calling tools from tools](#calling-tools-from-tools)), and `details` for UIs. A tool that declares `structuredOutputSchema` returns a matching `structuredOutput`, which the Harness validates. Programs calling a tool without one get its bounded output: one text item as a string, one image as its `ImageContent`, nothing as `""`, and anything else as the content list, for errors too. The transcript stores only `output` and `details`: + +```typescript +const lines = defineTool({ + name: "lines", + description: "Count the lines of a file", + parameters: Type.Object({ path: Type.String() }), + structuredOutputSchema: Type.Object({ lines: Type.Number() }), + execute: async (args, api, context) => { + const text = await readText(api, args.path, context); + const count = text.split("\n").length; + return { output: [{ type: "text", text: `${count} lines` }], structuredOutput: { lines: count } }; + }, +}); +``` + +A missing, undeclared, or invalid `structuredOutput` turns a nested call's result into an error result with an `invalid_structured_output` diagnostic; for a call the model made, which never sees it, it is dropped and reported through `onReport`. An error result may omit it; error results the Harness writes itself (blocked, invalid, interrupted, aborted) carry none, and their `diagnostics` say what went wrong. An `afterTool` hook that redacts a schema tool's `output` must redact its `structuredOutput` too. The built-in `bash` returns `{ output, truncated, fullOutputPath?, exitCode }`, where `output` is the retained tail the model sees, read with `api.retainedOutput()`, and answers a nonzero exit with an error result that still carries it. + A later extension's tool with the same name replaces an earlier one where both are selected, and `wrapTool()` decorates whichever tool won: ```typescript @@ -177,6 +196,52 @@ const Timing = defineExtension({ }); ``` +### Calling tools from tools + +A tool calls another tool with `api.executeTool()`, as a code mode script or an MCP bridge does. The nested call is its own tool task, owned by the calling one: it is validated, runs the `ToolTask` hooks, keeps its own output limits and replay policy, and returns its result to the caller instead of the transcript: + +```typescript +const testsPass = defineTool({ + name: "tests_pass", + description: "Run the test suite and say whether it passed", + parameters: Type.Object({}), + execute: async (_args, api, context) => { + const run = await api.executeTool("bash", { command: "npm test" }, context, { progress: false }); + // Absent when bash did not finish, for example on a timeout. + const exitCode = (run.structuredOutput as { exitCode: number } | undefined)?.exitCode; + return { output: [{ type: "text", text: exitCode === 0 ? "passed" : "failed" }] }; + }, +}); +``` + +- The caller gets the nested call's `taskId`, `structuredOutput`, `details`, `diagnostics`, `usage`, and `isError`, not the `output` meant for the model. A blocked, invalid, failed, or aborted nested call returns an `isError` result. `executeTool()` itself rejects once `execute()` has returned, when the caller is aborted, or when the Harness closes; cancelling its `context` stops only the wait. +- Hooks see a nested call's `call.parent`, so a guard can treat calls from scripts differently. +- While the caller runs, its nested calls show in `docs["pi.live"].nestedTools`, with the arguments they run with and a `summary` (error, duration, usage) once done, and as tool events with `parentToolCallId` and `parentTaskId`. `progress: false` keeps a nested call's running output out of `pi.live`. Before the caller settles, nested calls it left running are aborted. The caller's result message does not list its nested calls; their task records keep them. +- Each nested call has a key, its position among the caller's nested calls unless the caller passes `{ key }`. A replay-safe caller that reruns after a crash and calls in the same order, or passes the same keys, gets the nested calls it already made back, finished or still running, instead of starting them again. A nested call the crash interrupted before its result was committed follows its own `replay` policy, so a replay-safe tool with external effects still needs its own idempotency. A caller that is not replay-safe never reruns, so after a restart its unfinished nested calls are abandoned, with everything they own, before they run again. +- A nested call's `usage` is counted under its own tool; passing it on in the caller's result counts it twice. +- Nested results are not kept: they live in the caller's task documents (`NestedResultDoc`, one per nested call) only until the caller settles. The nested call's task record keeps only a small receipt. A stored result is exactly what `executeTool()` returns, without the model's `output`, so an image or a `bash` tail is stored once. +- Nothing bounds how deeply calls nest; a tool with the default `callers` may even call itself. Restrict `callers` where that matters (below). + +### Who may call a tool + +`callers` on a tool says who may call it: the model, other tools through `executeTool()`, or both, the default. A conversation's `modelTools` narrows which of its tools the model is offered, without taking them away from tools: + +```typescript +const codemode = defineTool({ name: "codemode", callers: ["model"], ... }); // scripts cannot start scripts +const search = defineTool({ name: "mcp__github__search", callers: ["tools"], ... }); // never offered to the model + +// The model is offered only codemode; its scripts call read, bash, and the MCP tools. +await root.configure({ modelTools: [codemode] }, context); +// The model is offered everything but bash, which scripts may still call. +await root.configure({ modelTools: { remove: [bash] } }, context); +``` + +`conversation.agent()` resolves both lists: `tools`, offered to the model, and `callable`, which nested calls resolve among. `tools` still decides what is enabled at all; `modelTools` cannot add to it. + +### Tasks a restart abandons + +A task that its creator awaits only in memory, and that the creator never resumes after a restart, can be created with `abandonOnRestart: true`. When a later Harness starts scheduling and finds it still live, it is aborted with `abortReason: "restart"`, together with everything it owns, before any of it runs again. Nested calls of tools that are not replay-safe get this automatically, and so do the child tasks such a tool creates with `api.createTask()`. A task abandoned this way whose definition is missing waits until its extension is installed, so its abort handler still cleans up. Its caller, and the conversation, wait with it; `harness.inspect()` shows it as blocked. Aborting it then orphans it, as any abort does. + ## System Prompt The system prompt is built from the selected extensions' sections, rendered in order before each request. A section sees the resolved agent, the environment built for the request, and committed documents: @@ -197,7 +262,8 @@ await root.configure( model: { provider: "openai", modelId: "gpt-6-sol" }, thinkingLevel: "high", extensions: { remove: [Coding] }, // edits the host default; an array selects exactly these, in order - tools: [readTool, bashTool], // an array offers exactly these; { remove: [...] } drops some + tools: [readTool, bashTool], // an array enables exactly these; { remove: [...] } drops some + modelTools: [readTool], // of those, offer the model only read; tools may still call bash instructions: "Only read; never edit files.", cwd: "/work/repo", }, @@ -440,7 +506,7 @@ const Subagent: Extension = defineExtension({ await api.details({ conversationId: child }, context); // lets a UI attach to the child const request = { type: "input", content: args.task, requestId: `subagent:${api.taskId}` } as const; const settled = await (await (await api.conversation(child, context))!.submit(request, context)).wait(context); - return { content: [{ type: "text", text: settled.status }] }; + return { output: [{ type: "text", text: settled.status }] }; }, }), ], @@ -571,6 +637,24 @@ registerStorageConformance({ describe, expect, it }, "My Storage", async (use) = The package root loads TypeBox, because the tool task validates arguments with pi-ai's `validateToolArguments()`. That costs about 23 MB of peak RSS unbundled, about 4 MB in a tree-shaken bundle. +## Errors + +Three rules decide what an error does: + +1. **A storage error ends the Harness.** The call that hit it rejects with the error. Everything after it, and every pending wait, rejects with `SessionFailed` (its `cause` is that error); watches and event streams end with `session_failed`; running tasks get their abort signal; `onReport` gets the error once. The Harness then closes itself. Nothing is retried; a custom storage retries its own transient failures. Reopen to recover: commits are atomic, so the store is consistent, and resubmitting with the same `requestId` finds a submission that was already admitted. +2. **An invalid request fails only that call:** a storage throws `StorageRequestError` for a bad cursor, an unknown conversation, or history a document does not keep. A read you cancelled fails only itself too. +3. **Your code fails only its own unit:** a throwing task phase faults that task, a throwing tool gets an error result, a throwing hook is reported and skipped (`beforeTool` blocks the call), and a throwing commit callback rolls back that commit. Listeners that throw, or return a rejected promise, are reported and the others still run. The exception is a host `RegistryReader` whose `snapshot()` throws: the Harness cannot schedule without it, so it fails the Harness like a storage error. + +`harness.closed` settles once the Harness has closed, by `close()` or after a storage error, so a host restarts on failure like this: + +```typescript +harness.closed.then((end) => { + if (end.reason === "failed") reopen(end.error); +}); +``` + +A failure closes the Harness by itself; `closed` settles once that close is done, even if the storage could not close cleanly. Closing waits for running task code to return, so code that ignores its abort signal delays `closed`. + ## Examples Runnable examples live in [`test/examples`](test/examples). Run one from this package directory with: @@ -608,7 +692,7 @@ Examples that call OpenAI need `OPENAI_API_KEY`; most use the faux provider othe - [`docs/pico-v5-handoff.md`](https://github.com/earendil-works/pi/blob/main/packages/durable/docs/pico-v5-handoff.md): the implementation plan - [`docs/pico-v5-chord-usage.md`](https://github.com/earendil-works/pi/blob/main/packages/durable/docs/pico-v5-chord-usage.md): how the package uses Chord -Benchmarks: `npm run bench:storage`, `npm run bench:storage:memory`, and `npm run bench:tool-output`. +Benchmarks: `npm run bench:storage`, `npm run bench:storage:memory`, `npm run bench:tool-output`, and `npm run bench:nested-tools`. ## License diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 83762e1cd..18d42868e 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -73,7 +73,7 @@ Required invariants: 7. The mutation line remains held through storage settlement and committed-state adoption. Commit/close observers run synchronously only to capture immutable state; document-state and watch user callbacks run later, off the line. -8. An uncertain storage failure is fatal to the open Session. It publishes +8. A storage error is fatal to the open Session (section 10.1). It publishes nothing and must be reopened. Preparation and checkpoint failures occur before storage admission and roll back normally. @@ -361,6 +361,7 @@ type HarnessOptions = { readonly conversationCreated?: (tx: Tx, conversation: ConversationRecord) => void | Promise; /** Wall clock for sleeps, retries, message timestamps, and task lifecycle times. Default `Date.now`. */ readonly now?: () => number; + /** Failures that do not fail the calling operation, and the Session's failure, once (section 10.1). */ readonly onReport?: (error: unknown) => void; }; @@ -434,8 +435,10 @@ type AgentState = { thinkingLevel?: ModelThinkingLevel; /** An array selects exactly these extensions, in order. An object edits the host default selection. */ extensions?: string[] | { add?: string[]; remove?: string[] }; - /** Filters the selected extensions' tools. An array offers exactly these, in order. */ + /** Filters the selected extensions' tools: the enabled tools. An array enables exactly these, in order. */ tools?: string[] | { remove: string[] }; + /** Filters the enabled tools the model may call: the tools it is offered. An array offers exactly these, in order. */ + modelTools?: string[] | { remove: string[] }; /** Rendered after every extension section, as the section `instructions`. */ instructions?: string; /** Directory within the environment's file system, passed to `HarnessOptions.env`. */ @@ -454,6 +457,7 @@ type AgentChange = { | { readonly add?: readonly Extension[]; readonly remove?: readonly Extension[] } | null; readonly tools?: readonly ToolRegistration[] | { readonly remove: readonly ToolRegistration[] } | null; + readonly modelTools?: readonly ToolRegistration[] | { readonly remove: readonly ToolRegistration[] } | null; readonly instructions?: string | null; readonly cwd?: string | null; }; @@ -466,7 +470,10 @@ type Agent = { readonly model?: ModelRef; readonly thinkingLevel: ModelThinkingLevel; readonly extensions: readonly Extension[]; + /** Offered to the model: enabled, `callers` includes `model`, and selected by `modelTools`. */ readonly tools: readonly Tool[]; + /** Nested calls resolve among these: enabled, and `callers` includes `tools`. */ + readonly callable: readonly Tool[]; /** Extension sections, then `instructions` when set. */ readonly sections: readonly PromptSection[]; readonly instructions?: string; @@ -1162,9 +1169,13 @@ without storage-assigned lifetime fields. There is no mutable `session.document()` API. ```ts +type SessionEnd = { readonly reason: "closed" } | { readonly reason: "failed"; readonly error: unknown }; + interface Session extends DocumentObserver { commit(change: (tx: Tx) => T | Promise, context: Context): Promise; close(context: Context): Promise; + /** Settles once closed, Storage included: `{ reason: "closed" }`, or `{ reason: "failed", error }` (section 10.1). */ + readonly closed: Promise; subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void; subscribeClose(listener: () => void): () => void; @@ -1368,7 +1379,7 @@ storage succeeds commit listeners, including conversation view mounts (section 9.3), capture it synchronously release the line; invoke listeners later storage fails - abort every prepared change, poison Session, and publish nothing + abort every prepared change, fail the Session (section 10.1), and publish nothing ``` A pending acquisition that resolves after sealing never exposes a draft; its @@ -1563,10 +1574,13 @@ line; document-state and watch user callbacks run later. External model, process, tool, network, and human effects run outside it. `subscribeCommits()` observes complete immutable publications synchronously on the line after adoption. `subscribeClose()` observes close synchronously when it -begins, after admission is sealed; the Harness stops watches and signals task -invocations there. Both return idempotent disposers; their listeners must not -throw, block, or call Session APIs. Document-state subscribers and watch -listeners still run later, off the line. +begins, after admission is sealed, also when a failure closes the Session; the +Harness ends waits, watches, and streams there, and signals task invocations +after every close listener has run, so what a task holds ends with the Session's +reason rather than as cancelled. Both return idempotent +disposers; their listeners must not block or call Session APIs, and one that +throws is reported without affecting the others (section 10.1). Document-state +subscribers and watch listeners still run later, off the line. ```ts await session.commit(async tx => { @@ -1642,6 +1656,10 @@ type TaskRecord = { readonly owner?: TaskId; readonly background: boolean; readonly abortRequested: boolean; + /** `restart` when the mark abandons the task after a restart, not an abort request (section 5.4). */ + readonly abortReason?: "restart"; + /** From `TaskOptions.abandonOnRestart`. Immutable. */ + readonly abandonOnRestart?: true; /** Wall clock at the first change to `running`; kept through waits and recovery. */ readonly startedAt?: number; /** Wall clock at the change to `terminal`. */ @@ -1707,6 +1725,8 @@ interface TaskRuntime extends DocumentObserver, Docum getTask(id: TaskId, context: Context): Promise | undefined>; /** Terminal receipt; rejects when the invocation ends. */ waitForTask(id: TaskId, context: Context): Promise>; + /** `abortTask()` of a task this task owns, resolving once it is terminal; rejects for a task another owns. */ + abortOwned(id: TaskId, context: Context): Promise; /** Outcomes of terminal tasks, in order; rejects when one is not terminal. Used after a wait (section 5.5). */ outcomes(ids: readonly TaskId[], context: Context): Promise[]>; /** Committed entry visible from the task's conversation. */ @@ -1747,6 +1767,8 @@ type TaskOptions = { readonly conversationId?: ConversationId; /** Conversation-owned tasks only: excluded from conversation abort and idle, and from cascades. */ readonly background?: boolean; + /** Its creator awaits it only in memory and does not resume after a restart (section 5.4). */ + readonly abandonOnRestart?: boolean; }; function defineTask( @@ -2043,7 +2065,30 @@ directly; otherwise the scheduler settles it when it would reserve the abort invocation, which is after the owned work drained (section 5.5), so an orphaned outcome never holds. Only an abort (direct, by conversation, or by cascade) orphans a task; a missing definition alone never -does. The orphaning commit performs the cleanup the task's code cannot: affected +does, and neither does a `restart` mark (below), whose task waits for its +definition, so its abort handler can still clean up. + +A task created with `abandonOnRestart` is awaited by its creator only in +memory, and the creator does not resume after a restart, as a replay-unsafe +tool does not. When such a task is still live once another Harness starts +scheduling (`resume()`, or the first submission or wait), it is pointless: the +first reservation pass sets its abort mark with `abortReason: "restart"` and +reserves nothing, so the next pass sees the marks. Opening alone marks nothing, +so an inspection-only open marks no task. The cascade passes the `restart` +reason on to the ordinary owned work below; any other cancellation intent +marks without a reason, and replaces a `restart` mark below it. A request +(`abortTask()`, a conversation abort) replaces a `restart` mark too, so a +restart-marked task waiting for its definition is then orphaned; an owner's own +cleanup through `runtime.abortOwned()` keeps the mark. Abort handlers see the +reason on the task record and can tell an abandonment from a request. + +No task below an owner with cancellation intent starts a run phase, even +before the cascade has marked it: reservation skips it, a running invocation +ends before its next phase, and both schedule the cascade again. Such work runs +only its abort handler. A reconciliation or reservation commit that fails fails +the Session (section 10.1). + +The orphaning commit performs the cleanup the task's code cannot: affected input submissions become unanswered with the reason, any matching active run control is cleared, and task-scoped documents retire. The terminal task record and unanswered submissions carry the reason; the only transcript entry written @@ -2064,9 +2109,8 @@ spend (the scheduler's commit has no task scope, so the entry has no `byTaskId`); its inputs become `unanswered` with reason `faulted` (detail: the error message) or the blocked reason; and `run`, `generation`, and `tools` are removed. Faults come from task bugs or malformed provider data, such as a -non-JSON value in a response, or a commit the Storage rejected without effect -(`StorageRejected`). An uncertain storage failure poisons the Session and writes -no outcome. For a `pi.tool` task it marks the task's tool slot `done` +non-JSON value in a response. A storage failure fails the Session and writes no +outcome (section 10.1). For a `pi.tool` task it marks the task's tool slot `done` without an entry; the run continues, and context derivation synthesizes the missing result (section 2.1). For a `pi.compaction` task it removes the task's compaction status (section 8.7). Outcomes a task commits for itself do their own settlement. @@ -2499,8 +2543,10 @@ extensions base = settings.extensions ?? every installed extension; in that ord duplicates keep their first position; names not installed are skipped tools the selected extensions' tools in extension order; a later same-name tool replaces an earlier one in place then every selected extension's tool wrappers, in extension order, then in each extension's order - then the filter: an array keeps exactly these names in its order, a repeated name at its first position; - { remove } drops these names + then the `tools` filter, which yields the enabled tools: an array keeps exactly these names in its order, + a repeated name at its first position; { remove } drops these names + `tools`: the enabled tools whose `callers` include "model" (default both), through the `modelTools` + filter, the same way; `callable`: the enabled tools whose `callers` include "tools" sections the selected extensions' sections, same-key replacement in place, then their section wrappers, then `instructions` when set hooks the selected extensions' hooks for the task's name, in extension order @@ -2616,16 +2662,19 @@ interface GenerationHooks { afterTools(assistant: EntryId, results: readonly EntryId[], api: HookApi, context: Context): void | Promise; } +/** `parent` is set for a nested call (section 7.3). */ +type ToolHookCall = ToolCall & { readonly parent?: { readonly taskId: TaskId; readonly callId: string } }; + interface ToolHooks { /** Before intent; replaces the arguments or blocks the call with error text. */ beforeTool( - call: ToolCall, + call: ToolHookCall, api: HookApi, context: Context, ): HookResult<{ readonly arguments?: JsonObject; readonly block?: string }>; - /** After execution, before the result entry; replaces the result. */ + /** After execution, before the result is committed; replaces the result. */ afterTool( - call: ToolCall, + call: ToolHookCall, result: ToolExecutionResult, api: HookApi, context: Context, @@ -2704,8 +2753,10 @@ type ToolDiagnostic = { readonly code?: string; }; +/** Three channels: `output` for the model, `structuredOutput` for programs that call the tool, `details` for UIs. */ type ToolExecutionResult = { - readonly content?: ToolResultMessage["content"]; + readonly output?: ToolResultMessage["content"]; + readonly structuredOutput?: JsonValue; readonly isError?: boolean; readonly details?: TDetails; readonly diagnostics?: readonly ToolDiagnostic[]; @@ -2734,6 +2785,8 @@ interface ToolExecutionApi extends Docum /** Built by `HarnessOptions.env` for this call. */ readonly env: ExecutionEnv | undefined; output(chunk: string | Uint8Array): void; + /** The output retained so far, as the model will see it, and whether earlier output was dropped. */ + retainedOutput(): { readonly text: string; readonly truncated: boolean }; diagnostic(diagnostic: ToolDiagnostic): void; details(value: TDetails, context: Context): Promise; commit(change: (tx: Tx) => T | Promise, context: Context): Promise; @@ -2748,14 +2801,38 @@ interface ToolExecutionApi extends Docum getTask(id: TaskId, context: Context): Promise | undefined>; waitForTask(id: TaskId, context: Context): Promise>; conversation(id: ConversationId, context: Context): Promise; + /** Run another tool as a nested call of this one (below) and wait for its result. */ + executeTool( + name: string, + args: JsonObject, + context: Context, + options?: { readonly key?: string; readonly progress?: boolean }, + ): Promise; } +/** A nested call's result, for programs: no output for the model, and `control` does not apply. */ +type NestedToolExecutionResult = { + /** The nested call's tool task. */ + readonly taskId: TaskId; + /** A schema tool's validated `structuredOutput`, else its bounded output, collapsed (below). */ + readonly structuredOutput?: JsonValue; + readonly isError: boolean; + readonly details?: JsonValue; + readonly diagnostics: readonly ToolDiagnostic[]; + readonly usage?: Usage; + readonly durationMs?: number; +}; + type ToolRegistration = Tool & { readonly replay?: "safe" | "unsafe"; readonly executionMode?: ToolExecutionMode; /** Pure repair of commonly malformed arguments; runs before validation, which still checks its result. */ prepareArguments?(args: unknown): Static; + /** Schema of `structuredOutput`; omitted, programs get the bounded output, collapsed (section 7.3). */ + readonly structuredOutputSchema?: TSchema; + /** Who may call the tool; default both. */ + readonly callers?: readonly ("model" | "tools")[]; readonly outputLimits?: { readonly maxBytes?: number; readonly maxLines?: number; @@ -2784,6 +2861,112 @@ and `retain: "head"`. Omitted `executionMode` follows the settings' `toolExecution`; one `sequential` call makes its whole round sequential (section 8.3). +A running tool calls another tool with `api.executeTool()`, as a codemode +script or an MCP bridge does. The nested call is its own `pi.tool` task, owned +by the calling tool's task, so it has its own task ID for idempotence and +ownership, runs the whole tool pipeline (section 8.4) including the `ToolTask` +hooks, which see `call.parent`, and reports progress in `pi.live.nestedTools` +(section 8.2). Its result returns to the caller as a `NestedToolExecutionResult` and +never enters the transcript; the model sees only what the caller puts in its +own result. Its tool resolves among the agent's `callable` tools: the enabled +tools whose `callers` include `tools` (section 7.1). A call that is +unavailable, invalid, blocked, throws, is interrupted, or is aborted returns +an `isError` result; a nested task the scheduler faulted or orphaned returns a +synthesized one. + +The nested call's key names it within its caller; the nested call ID is +`/`. By default the key is the call's position among the +invocation's nested calls, `1`, `2`, ... An explicit `key` must be non-empty, +without `/`, not `__proto__`, and not a positive integer, so call IDs never +collide. One commit creates the nested task, its slot, and an index entry from +the key to the task in `pi.tool.nested`, a task-scoped document of the caller, +which retires with it. A caller that is not replay-safe creates its nested +calls, and the child tasks it creates through `api.createTask()` unless the +options say otherwise, with `abandonOnRestart` (section 5.4): it never resumes +after a restart, so neither do they. With `progress: false`, a nested call +commits only its status to its slot, no running output, details, or +diagnostics; its result still carries its details and diagnostics, and a +schema-less tool's output as `structuredOutput`, but an interrupted or aborted +call, whose result is built from its slot, then has none of them. Programs +never get the partial output of an interrupted or aborted call. Nothing bounds how +deeply calls nest; `callers` keeps a tool from being called by tools at all. + +The nested call's result does not go into its receipt, which stays +`{ kind: "nested" }`. Its terminal commit writes the result to +`NestedResultDoc` (`pi.tool.nested-result`), a task-scoped document family of +the caller keyed by the nested task ID, one member per result, and sets its +slot's `summary` (`isError`, `durationMs`, `usage`, bounded error text). The +stored result is exactly what `executeTool()` returns: it keeps no `output`, +which only the model reads, so each result, an image or a `bash` tail, is stored +once. The caller is +never terminal before its owned nested calls, so the write always finds its +documents open; a nested call missing from the caller's index throws. The +caller reads the result after the wait, and every nested result is gone from +storage once its caller is terminal. A nested call the scheduler faulted or +orphaned has no stored result; the caller gets a synthesized `isError` one. + +A later `executeTool()` with a used key, from the same invocation or from a +replay-safe rerun after recovery that calls in the same order or passes the +same `key`, waits for that task again, finished or still running, instead of +creating another; a different tool name or arguments that +are not structurally equal throws. So a replay-safe caller admits each nested +call once and reuses its committed result. This is not exactly-once execution: +a nested call interrupted by a crash before its result commits follows its own +replay policy, rerunning when safe and returning `interrupted` otherwise, so a +replay-safe tool with external effects still needs its own idempotency. Nested calls start when +made, whatever their `executionMode`; the caller decides what runs at once by +what it awaits. A nested result's `usage` is counted under its tool when it +settles (section 8.6); a caller that passes it on in its own result counts it +again. + +Before a caller settles, by any path, it lets nested call admissions its +invocation started commit, then aborts every nested call it left running with +`runtime.abortOwned()`, all at once, and waits for them, so they settle into +their slots before the caller's settlement removes them. `executeTool()` throws +once `execute()` has returned, and rejects when the caller is aborted or the +Harness closes; cancelling its `context` stops only the wait. An abort that +arrives during this cleanup still aborts the caller, whose finished result is +then replaced by the `aborted` one. The caller's result message lists none +of its nested calls (no pi-ai `nestedCalls`): their task records keep every +call and its arguments, and a tool that wants a list in its result, as code +mode does, writes one into its own `details`. A crash between the +caller's return and its result commit leaves it in `execute`: a replay-safe +caller runs again and finds its nested calls by key; any other caller settles +`interrupted`, and its nested calls were abandoned before they could run. + +Programs that call a tool get its `structuredOutput`. A tool that declares +`structuredOutputSchema` must return a `structuredOutput` matching it, except +on an error result, where it may omit it; the Harness validates it after the +`afterTool` chain, like arguments. A tool without a schema must not return one; +programs get its bounded output, the content the model sees without the +rendered diagnostics, for a success and for an error alike: one text item as +its string, one image as its `ImageContent`, no items as `""`, and anything +else as the content list. So a +hook that redacts a schema-less tool's output redacts what programs get. An +error result the Harness writes itself (unavailable, invalid arguments, +blocked, interrupted, aborted, abandoned, or a scheduler fault) has no +`structuredOutput`; its diagnostics say what went wrong, as they do for a +schema tool's error result without one. A nested call whose result breaks this gets an error result +with an `invalid_structured_output` diagnostic, without the broken value: a +schema tool's then has no `structuredOutput`, a schema-less tool's its output; +a model-issued call, whose `structuredOutput` the model never sees, only loses +it, and the break is reported through `HarnessOptions.onReport`. `structuredOutput` has no size bound; +keeping it small is the tool's job, as `bash` keeps its retained tail and the +spill path rather than the whole output. A model-issued call's result entry +does not store it, and the Harness does not build a schema-less one for it; a +nested call's result does, until its caller settles. A +tool that returns the text it streamed in its `structuredOutput` reads it with +`api.retainedOutput()`, the bounded text the model sees when the result omits +`output`. + +`callers` says who may call a tool: the model, other tools through +`executeTool()`, or both (the default). A code mode tool is `["model"]`, so +scripts cannot start scripts; tools behind code mode, such as MCP tools, can be +`["tools"]`, so the model is never offered them. A conversation's `modelTools` +narrows which of its model-callable tools the model is offered, for example +only the code mode tool, while the others stay callable by tools. Neither can +widen what `tools` enables. + Tools reach files and processes only through `api.env`, never through an environment captured when the tool was built. The tool task builds it for each call with `HarnessOptions.env` from the conversation's ID and `cwd` (section @@ -2797,7 +2980,7 @@ A running tool reports output and details to the UI, mirroring the two halves of its final result, plus diagnostics (below): - `output(chunk)` appends running text output, like stdout. If `execute()` omits - `content`, the final retained output becomes one text content item; no output + `output`, the final retained output becomes one text content item; no output becomes an empty content list. Retained output is an exact slice of whole lines of the stream (the first lines for `head`, the last for `tail`), trailing newline included; a single line longer than `maxBytes` is cut at the byte limit @@ -2855,8 +3038,11 @@ renderer needs it. The environment's shell streams raw output chunks and spills the complete output to a file once it crosses byte or line thresholds; it keeps no bounded view of its own, so `output()` is the one place output is bounded, sanitized, and throttled. The `bash` tool pipes those chunks into `output()`, -reports the spill path as a diagnostic, and throws on a nonzero exit or timeout; -the error result still carries the retained output and diagnostics. +reports the spill path as a diagnostic, returns `structuredOutput: { output, +truncated, fullOutputPath?, exitCode }` with the retained output, answers a +nonzero exit with an error result carrying an `exit_code` diagnostic, and +throws on a timeout; either error result still carries the retained output and +diagnostics. An environment that moves output over a slow link, such as one on another host, need not move all of it. `api.outputWindow` names the tail a tail-retaining @@ -2920,7 +3106,7 @@ final flush and settles any `details()` promise still pending. Abort and close o invocation and Session admission gates: uncommitted buffered updates may be discarded, while admitted commits settle. Cancellation, callback, tracker preparation, and checkpoint failures occur before Storage admission and do not -poison the Session. An uncertain Storage failure follows the fatal Session rule. +fail the Session. A Storage failure follows the fatal Session rule (section 10.1). Tools come with extensions (section 7.1) and have a name, description, JSON schema, replay policy, and execute function. `wrapTool(tool, wrapper)` decorates @@ -3024,7 +3210,7 @@ export const Subagent = defineExtension({ const request = { type: "input", content: task, requestId: `subagent:${api.taskId}` } as const; const settled = await (await handle.submit(request, context)).wait(context); if (settled.status !== "done" || settled.type !== "input") throw new Error(`Subagent failed: ${settled.status}`); - return { content: [{ type: "text", text: await answerText(api, settled.answer, context) }] }; + return { output: [{ type: "text", text: await answerText(api, settled.answer, context) }] }; }, })], }); @@ -3358,6 +3544,29 @@ type LiveState = { /** Provider-side deferred response being polled. */ deferred?: { pollAt: number }; }; + /** + * Nested calls of running tool calls (section 7.3), in creation order, so a parent precedes its children and task + * IDs ascend. + */ + nestedTools?: { + /** `/`: the call's ID in tool events, unique because keys contain no `/`. */ + callId: string; + parentCallId: string; + /** The calling tool task, model-issued or nested; the Harness relates slots by task ID. */ + parentTaskId: TaskId; + taskId: TaskId; + name: string; + /** The call's arguments, unbounded; once it executes, those it runs with, after repair, hooks, and coercion. */ + arguments: JsonObject; + status: "pending" | "running" | "done"; + output?: string; + droppedBytes?: number; + droppedLines?: number; + details?: JsonValue; + diagnostics?: ToolDiagnostic[]; + /** Once done: how the call ended, with up to 500 characters of error text; the result is not here (section 7.3). */ + summary?: { isError: boolean; durationMs?: number; usage?: Usage; error?: string }; + }[]; /** The current tool round in call order, from the tool-calling answer until the generation's `tools` phase ends it. */ tools?: { callId: string; @@ -3400,7 +3609,7 @@ type CompactionStatus = { | version | `1` | | scope/history/fork | conversation, `latest`, `initial` | | `initial()` | `{}` | -| checkpoint | complete base whenever nothing runs: `generation` absent and no `running` tool slot | +| checkpoint | complete base whenever nothing runs: `generation` absent and no `running` tool or nested slot | | view mount | `docs["pi.live"]` | | created | with every Harness conversation (section 2.2) | @@ -3425,6 +3634,16 @@ The generation's `tools` phase removes `tools`. Slot updates apply only while a `taskId` exists; without one, the durable partial output, details, and diagnostics are empty. +A nested slot is pushed to `nestedTools` in the commit that creates its task, +with the arguments the caller passed, runs like a tool slot, gets the arguments +it executes with in its intent commit when they differ (a model-issued call's +are in its assistant entry), and in its terminal commit becomes `done`, with its progress +removed like a tool slot's and its `summary` set, enough for a status line; its +result is in the caller's `NestedResultDoc`, never in `pi.live` (section 7.3). When any call settles, its terminal commit +removes the nested slots below it, transitively by `parentTaskId`; its +left-running nested calls were aborted first. `endRun` and the generation's +`tools` phase remove `nestedTools` with `tools`. + A compaction's status is added in the commit that creates the task and removed in the commit that decides its outcome: the task's own outcome commit, even when that outcome holds as `completing` (section 5.5), or the scheduler's cleanup @@ -3602,15 +3821,26 @@ like the abort handler. ### 8.4 Tool ```ts -type ToolTaskInput = { assistant: EntryId; callId: string }; +type ToolTaskInput = + | { kind: "model"; assistant: EntryId; callId: string } + | { kind: "nested"; parent: TaskId; parentCallId: string; key: string; call: ToolCall; progress?: false }; type ToolTaskCheckpoint = | { phase: "call" } | { phase: "execute"; arguments: JsonObject; replay: "safe" | "unsafe" }; -type ToolTaskResult = { entryId: EntryId; control?: ToolControl }; +type ToolTaskResult = { kind: "model"; entryId: EntryId; control?: ToolControl } | { kind: "nested" }; ``` -`pi.tool` is version 1 and starts at `{ phase: "call" }`. The input stays small -because the terminal record keeps it; the call is read from the assistant entry. +`pi.tool` is version 2 and starts at `{ phase: "call" }`; `migrate` gives a +version 1 input `kind: "model"`, and readers treat a terminal result without +`kind` as a model-issued call's. A model-issued call's input stays small +because the terminal record keeps it; the call is read from the assistant +entry. A nested call (section 7.3) carries its call, since no entry holds it, +and settles differently: its result commit records the result's `usage` +(section 8.6), writes its result for programs to the caller's +`NestedResultDoc` (section 7.3), marks its nested slot `done` with its +summary, appends no entry, and ends with `{ kind: "nested" }`; `control` does +not apply. +Below, `{ entryId }` is a model-issued call's result. - `call` reads the call with `runtime.entry()`, resolves the tool among the phase's agent `tools` (section 7.3), validates @@ -3657,11 +3887,11 @@ Otherwise it reads the tool records with `runtime.getTask()` and the round's result entries from the `pi.live.tools` slots, runs the `afterTools` observers, and then commits once: -- It edits the conversation's stored `pi.agent` `tools` for every `addTools` - name: an array gets the name appended unless it holds it already, and - `{ remove }` loses the name. With `tools` unset, every tool is already - offered, and nothing is written. The next preparation offers the tool when it - resolves. +- It edits the conversation's stored `pi.agent` `tools` and `modelTools` for + every `addTools` name: an array gets the name appended unless it holds it + already, and `{ remove }` loses the name. An unset filter already lets every + tool through, and nothing is written for it. The next preparation offers the + tool when it resolves and the model may call it. - When every result of the round requests `terminate`, or any requests `handoff`, it appends the handoff's `pi.reset` entry, if any, settles the run's inputs `done` with the tool-calling answer, removes `run` and `tools`, @@ -3704,7 +3934,8 @@ type UsageState = { `pi.usage` is the ledger of the conversation's own spend. Every built-in writer of a `pi.assistant` entry adds its message's `usage` to `models` under the message's own `provider/model`, and every writer of -a `pi.tool-result` entry with `usage` adds it to `tools`, in the same commit. +a `pi.tool-result` entry with `usage` adds it to `tools`, in the same commit, as +does the result commit of a nested call (section 7.3) with `usage`. Every summarization attempt of a compaction task adds its response's `usage` to `models` in the commit that classifies it (section 8.7); that spend has no entry, whether the summary is placed, fails, or ends stale. Failed and aborted attempts @@ -3922,7 +4153,9 @@ type DocumentState = type WatchEnd = | { readonly reason: "stopped" | "cancelled" | "session_closed" | "retired" } - | { readonly reason: "listener_error"; readonly error: Error }; + | { readonly reason: "listener_error"; readonly error: Error } + /** The Session failed; `error` is what failed it, usually a storage error (section 10.1). */ + | { readonly reason: "session_failed"; readonly error: unknown }; interface WatchHandle { /** Acquisition revision before start; latest delivered immutable revision afterward. */ @@ -4094,6 +4327,19 @@ covers exactly one conversation (not its owned subtree), and emits an event only after the commit that makes it true. Its protocol may change without notice. ```ts +/** + * Which call a tool event is about. `toolCallId` is the provider's ID for a model-issued call and + * `/` for a nested one; consumers match it and never parse it. `taskId` is the call's tool task, + * absent for a call that never got one. A nested call also names its caller by call ID and by tool task. + */ +type ToolEventCall = { + toolCallId: string; + toolName: string; + taskId?: TaskId; + parentToolCallId?: string; + parentTaskId?: TaskId; +}; + type AgentEvent = | { type: "snapshot"; @@ -4102,6 +4348,8 @@ type AgentEvent = /** Current generation attempt: its in-flight partial, retry backoff, or deferred poll. */ generation?: { attempt: number; message?: AssistantMessage; retry?: { at: number; error: string }; deferred?: { pollAt: number } }; tools: readonly ToolSlot[]; + /** `pi.live.nestedTools` (section 8.2). */ + nestedTools: readonly NestedToolSlot[]; /** `pi.live.compactions` (section 8.2). */ compactions: readonly CompactionStatus[]; inbox: readonly { id: SubmissionId; mode: InboxItem["mode"] }[]; @@ -4116,18 +4364,16 @@ type AgentEvent = | { type: "message_start"; message: Message } | { type: "message_update"; usage: Usage; changes: readonly MessageChange[] } | { type: "message_end"; entry: EntryRecord } - | { type: "tool_execution_start"; toolCallId: string; toolName: string; args: JsonObject } - | { + | ({ type: "tool_execution_start"; args: JsonObject } & ToolEventCall) + | ({ type: "tool_execution_update"; - toolCallId: string; - toolName: string; /** A front trim and then an append of the retained window, or its replacement. */ output?: { trimStart?: number; append?: string } | { set: string }; details?: JsonValue; diagnostics?: readonly ToolDiagnostic[]; - } - /** `entry` is absent when the tool task faulted or was orphaned. */ - | { type: "tool_execution_end"; toolCallId: string; toolName: string; entry?: EntryRecord } + } & ToolEventCall) + /** A model-issued call ends with its `entry`, a nested call with its `result`; see below for when neither. */ + | ({ type: "tool_execution_end"; entry?: EntryRecord; result?: NestedToolExecutionResult } & ToolEventCall) | { type: "inbox_update"; items: readonly { id: SubmissionId; mode: InboxItem["mode"] }[] } | { type: "submission"; record: SubmissionRecord } | { type: "auto_retry_start"; attempt: number; at: number; errorMessage: string } @@ -4199,6 +4445,12 @@ Events derive from committed changes: orphan. An unfinished slot that disappears because its run ended ends with the result entry appended in the same commit, as for the unstarted calls of an aborted round, directly before its `message_start`, or without an entry. + Nested slots produce the same events with `parentToolCallId`; a nested call + ends with its `result`, taken from the `NestedResultDoc` value its terminal + commit published, or without one after a fault or orphan or when its slot + disappears unfinished. Snapshots show done nested slots with their summary, + not their result; a late subscriber reads a live caller's results from its + `NestedResultDoc`. - `inbox_update`, `agent_changed`, `usage_changed`: the document changed; a retired one reads as its initial value, as in a snapshot. Registry installs and settings changes make no commit and emit nothing; a UI showing resolved @@ -4216,7 +4468,8 @@ One commit produces one batch, in this order: `tool_execution_start`, and retry/deferred events; then entries in append order with their `message_start`/`message_end` or `entry_appended`, where a tool result's `tool_execution_end` directly precedes its `message_start`, as in the coding -agent; then the `tool_execution_end` of tools ending without an entry; +agent, and the `tool_execution_end` of nested calls, children first, precede +the entries; then the `tool_execution_end` of tools ending without an entry; then `compaction_end`, `task_failed`, `turn_end`, `run_end`; then `submission` events in ID order; then `inbox_update`, `agent_changed`, and `usage_changed`; and last `compaction_start`, `run_start`, and `turn_start`. A stream buffers at most 100 undelivered @@ -4417,7 +4670,10 @@ type StorageWrite = */ interface Storage { commit(writes: readonly StorageWrite[], context: Context): Promise; - /** Allocate from the one global numeric namespace; the generic brand is compile-time only. */ + /** + * Allocate from the one global numeric namespace, above every ID minted or stored before, also across reopen: scans + * order by ID as creation order, and the Harness relies on it. The generic brand is compile-time only. + */ mintId>(): Promise; conversation(id: ConversationId, context: Context): Promise; @@ -4443,11 +4699,10 @@ interface Storage { } ``` -`StorageRejected` means a batch was rejected before any durable effect and is -guaranteed not to have committed. Session rolls such a batch back normally; -unknown failures after Storage admission remain fatal because their commit state -is uncertain. Backends use `StorageRejected` for deterministic `document.copy` -source, replay, and consistency failures only when rollback is guaranteed. +Every error a method throws is final for the Session (section 10.1), so a +backend retries its own transient failures. A read throws `StorageRequestError` +for an invalid request, with no durable effect; from `commit()` it is fatal like any +other error. A `document.copy` reads committed pre-batch source state independent of command order. The source must be an alive conversation document at the selected point, @@ -4514,6 +4769,73 @@ a stored version boundary; a version transition must be a base. The semantic conformance suite covers memory, SQLite, and JSONL. +### 10.1 Failures + +One rule per kind of error, so a host knows what happens without reading code. + +**A storage error ends the Session.** Any error a `Storage` method throws, a +commit, a read, or `mintId()`, fails the Session, with two exceptions below. +Neither the Session nor the Harness retries anything: a Storage retries its own +transient failures inside the method. Reopening recovers from what was committed, +which is always consistent, because commits are atomic; a call that threw may or +may not have committed, so a host that resubmits after reopening reuses its +`requestId`, which finds an admitted submission. On the first failure: + +- The call that hit it rejects with that error. Every later call, every pending + wait (`submission.wait()`, `waitForTask()`, `waitForIdle()`, `abortTask()`), + and every wait that starts later rejects with `SessionFailed`, whose `cause` is + the first error; so does every Storage call still underway when it returns, + whatever it returned. +- Watches and event streams end with `{ reason: "session_failed", error }`; + attached replicated states stop updating and keep their last value. +- Running task invocations get their abort signal. `HarnessOptions.onReport` + gets the error once, after the Session is sealed, so a report handler that + calls back finds it failed. +- The Session closes itself. `closed` settles `{ reason: "failed", error }` once + that close has run, also when the backend could not close cleanly; after a + plain `close()` it settles + `{ reason: "closed" }`. Closing the backend waits for every Storage call + underway, refusing new ones, and for running task code to return: code that + ignores its abort signal keeps the backend open and `closed` pending. +- A failing `Storage.close()` fails the Session too, unless it already failed; + the `close()` call rejects with it, and `closed` settles `failed`. Waits and + watches had already ended as for a close (`session_closed`). + +The exceptions fail only their call, and apply to reads alone: a +`StorageRequestError`, which a Storage throws for an invalid read, with no +durable effect (an unknown conversation, a malformed cursor or one from another scan +order, history a document does not keep), and a read rejected because its +caller's context was aborted. An error from `commit()` or `mintId()` is never +exempt: once a batch is admitted, whether it committed is unknown. + +A commit callback that catches a failed read cannot commit afterwards: the Session +checks its health after the callback. Adopting a committed batch in memory that +fails also fails the Session, since memory would be behind storage. + +**The Session's own bookkeeping fails it too.** A throw in a commit the +scheduler makes for itself (reconciliation, reservation, a fault or terminal +write) fails the Session. The only extension code there is a task's `migrate`, +whose throw is caught and blocks that task; anything else that throws is a +storage failure, a host callback such as a `RegistryReader` whose `snapshot()` +throws, or a bug, and running the commit again fixes none of them. A throw in a commit +listener of the Session's own components (scheduler, submissions, views, task +graph, document observers) fails it as well, since their memory would fall +behind storage; the commit itself stands. + +**User code fails only its unit.** A task phase that throws ends that task +`faulted`; a throwing `migrate` blocks it as `migration_failed`; a tool that +throws, or whose environment cannot be built, gets an error result; hooks that +throw are reported and skipped, except `beforeTool`, whose throw blocks the call; +a commit callback, `conversationCreated`, `init`, or a document's `initial`, +`migrate`, or `checkpointWhen` that throws rolls its commit back and rejects it. +Callbacks that must not fail their caller are contained: a host +`subscribeCommits()` or `subscribeClose()` listener, a watch listener, a +replicated-state subscriber, and a view observer that throws, or returns a +promise that rejects, are reported, and the others still run; a watch whose +listener throws ends with `listener_error`. A throwing or rejecting `onReport` is +dropped, since nothing is left to report it to. A throwing `now()` is reported +once, and `Date.now` serves instead. + ## 11. Backends ### 11.1 Memory @@ -4724,8 +5046,9 @@ These are contracts, not invitations to add defensive machinery: - **Services outliving the Harness:** withdraw Chord services and detach clients before closing the Harness. A state ended by close keeps its last value and never updates again. -- **Fatal storage errors:** after an uncertain storage failure the Session is - poisoned. Do not catch the error and continue using it. +- **Fatal storage errors:** after a storage error the Session has failed and + closed itself; reopen it (section 10.1). Retrying the failed call cannot + succeed: retry transient failures inside the Storage. - **JSONL durability:** default JSONL ordering handles ordinary process crashes; without durable mode it does not promise acknowledged commits survive power or host failure. diff --git a/packages/durable/package.json b/packages/durable/package.json index a2b7288f7..a26b29117 100644 --- a/packages/durable/package.json +++ b/packages/durable/package.json @@ -87,6 +87,8 @@ "bench:storage": "vitest bench --config vitest.benchmark.config.ts", "bench:storage:memory": "node --conditions=source --experimental-strip-types test/storage-memory.ts", "bench:tool-output": "node --conditions=source --experimental-strip-types --expose-gc test/tool-output-bench.ts", + "bench:nested-tools": "node --conditions=source --experimental-strip-types --expose-gc test/nested-tools-bench.ts", + "bench:nested-images": "node --conditions=source --experimental-strip-types --expose-gc test/nested-images-bench.ts", "prepublishOnly": "npm run clean && npm run build" }, "keywords": [ diff --git a/packages/durable/src/errors.ts b/packages/durable/src/errors.ts index 65f1572b8..c66a1cee2 100644 --- a/packages/durable/src/errors.ts +++ b/packages/durable/src/errors.ts @@ -1,5 +1,18 @@ import type { ConversationId } from "./types.ts"; +/** + * The message of a thrown value, without throwing itself: `String()` throws for values such as `Object.create(null)`, + * and code that formats an error must not fail for the value it formats. + */ +export function errorMessage(error: unknown): string { + if (error instanceof Error) return error.message; + try { + return String(error); + } catch { + return Object.prototype.toString.call(error); + } +} + /** A transaction read a table after its first table write. Read every required row before writing. */ export class ReadAfterWrite extends Error { constructor(method: string) { @@ -8,11 +21,27 @@ export class ReadAfterWrite extends Error { } } -/** Storage rejected a batch before any durable effect; the owning Session may continue safely. */ -export class StorageRejected extends Error { - constructor(message: string, options?: ErrorOptions) { - super(message, options); - this.name = "StorageRejected"; +/** + * A Storage read rejected an invalid request, such as an unknown conversation or a cursor from another scan, with no + * durable effect. Unlike any other error a Storage throws, it fails only that read, not the Session; from `commit()` it is + * fatal like any other. + */ +export class StorageRequestError extends Error { + constructor(message: string) { + super(message); + this.name = "StorageRequestError"; + } +} + +/** + * The Session failed, so nothing runs or commits any more. It fails when a Storage method throws, a commit cannot be + * adopted, or the Harness's own bookkeeping throws (a scheduler commit, an internal commit listener, a registry whose + * `snapshot()` throws). `cause` is that first error. Reopen the Session; reopening recovers from what was committed. + */ +export class SessionFailed extends Error { + constructor(cause: unknown) { + super("Session failed after a storage error; close and reopen it", { cause }); + this.name = "SessionFailed"; } } diff --git a/packages/durable/src/harness/agent.ts b/packages/durable/src/harness/agent.ts index 6908b957d..b52d85943 100644 --- a/packages/durable/src/harness/agent.ts +++ b/packages/durable/src/harness/agent.ts @@ -79,19 +79,22 @@ export async function configure(tx: Tx, conversationId: ConversationId, change: } /** - * `addTools` of a tool round: an array gets each name it lacks appended, `{ remove }` loses the names, and unset tools - * already offer every tool, so nothing is written. + * `addTools` of a tool round, applied to `tools` and `modelTools`, so the next request offers the tools: an array gets + * each name it lacks appended, `{ remove }` loses the names, and an unset filter already lets every tool through, so + * nothing is written. */ export async function addTools(tx: Tx, conversationId: ConversationId, added: readonly string[]): Promise { const state = await tx.doc(AgentDoc, conversationId); - const tools = state.tools; - if (tools === undefined) return; - if (Array.isArray(tools)) { - for (const name of added) if (!tools.includes(name)) tools.push(name); - } else { - const remove = (tools as { remove: string[] }).remove; - if (remove.some((name) => added.includes(name))) { - state.tools = { remove: remove.filter((name) => !added.includes(name)) }; + for (const field of ["tools", "modelTools"] as const) { + const filter = state[field]; + if (filter === undefined) continue; + if (Array.isArray(filter)) { + for (const name of added) if (!filter.includes(name)) filter.push(name); + } else { + const remove = (filter as { remove: string[] }).remove; + if (remove.some((name) => added.includes(name))) { + state[field] = { remove: remove.filter((name) => !added.includes(name)) }; + } } } } @@ -116,11 +119,17 @@ function applyChange(state: Draft, change: AgentChange): void { ...(extensions.remove === undefined ? {} : { remove: names(extensions.remove) }), }, ); - const tools = change.tools; - set( - "tools", - tools === undefined || tools === null ? tools : isList(tools) ? names(tools) : { remove: names(tools.remove) }, - ); + for (const field of ["tools", "modelTools"] as const) { + const filter = change[field]; + set( + field, + filter === undefined || filter === null + ? filter + : isList(filter) + ? names(filter) + : { remove: names(filter.remove) }, + ); + } set("instructions", change.instructions); set("cwd", change.cwd); } @@ -194,19 +203,11 @@ export function resolveAgent( } } - const filter = state?.tools; - let tools: Tool[]; - if (filter === undefined) tools = [...composed.values()]; - else if (Array.isArray(filter)) { - tools = []; - for (const name of new Set(filter)) { - const tool = composed.get(name); - if (tool !== undefined) tools.push(tool); - } - } else { - const removed = new Set((filter as { remove: string[] }).remove); - tools = [...composed.values()].filter((tool) => !removed.has(tool.name)); - } + const enabled = filterTools([...composed.values()], state?.tools); + const callableBy = (caller: "model" | "tools") => (tool: Tool) => + tool.callers === undefined || tool.callers.includes(caller); + const tools = filterTools(enabled.filter(callableBy("model")), state?.modelTools); + const callable = enabled.filter(callableBy("tools")); const instructions = state?.instructions; const agentSections = [...sections.values()]; @@ -217,6 +218,7 @@ export function resolveAgent( thinkingLevel: state?.thinkingLevel ?? "off", extensions, tools, + callable, sections: agentSections, ...(instructions === undefined ? {} : { instructions }), ...(state?.cwd === undefined ? {} : { cwd: state.cwd }), @@ -224,6 +226,20 @@ export function resolveAgent( return agent; } +/** `tools` through a stored filter: an array selects exactly its names, in its order; `{ remove }` drops names. */ +function filterTools(tools: Tool[], filter: AgentState["tools"]): Tool[] { + if (filter === undefined) return tools; + if (Array.isArray(filter)) { + const byName = new Map(tools.map((tool) => [tool.name, tool])); + return [...new Set(filter)].flatMap((name) => { + const tool = byName.get(name); + return tool === undefined ? [] : [tool]; + }); + } + const removed = new Set((filter as { remove: string[] }).remove); + return tools.filter((tool) => !removed.has(tool.name)); +} + /** Selected installed extensions: the stored array, or the default selection edited by `{ add, remove }`. */ function selectExtensions( stored: AgentState["extensions"], diff --git a/packages/durable/src/harness/events.ts b/packages/durable/src/harness/events.ts index dc14df949..080ed6254 100644 --- a/packages/durable/src/harness/events.ts +++ b/packages/durable/src/harness/events.ts @@ -15,8 +15,9 @@ import type { } from "../types.ts"; import { AgentDoc } from "./agent.ts"; import type { InboxItem, InboxState } from "./inbox.ts"; -import type { CompactionStatus, LiveState, ToolSlot } from "./live.ts"; -import type { AgentState, CompactionReason, Harness, ToolDiagnostic } from "./types.ts"; +import type { CompactionStatus, LiveState, NestedToolSlot, ToolSlot } from "./live.ts"; +import { NestedResultDoc } from "./tool.ts"; +import type { AgentState, CompactionReason, Harness, NestedToolExecutionResult, ToolDiagnostic } from "./types.ts"; import { UsageDoc, type UsageState } from "./usage.ts"; import { scanAll } from "./util.ts"; import { type ConversationView, conversationViews } from "./view.ts"; @@ -44,6 +45,8 @@ export type SnapshotEvent = { deferred?: { pollAt: number }; }; tools: readonly ToolSlot[]; + /** `pi.live.nestedTools`: nested calls of running tool calls. */ + nestedTools: readonly NestedToolSlot[]; /** `pi.live.compactions`: live compactions with their attempt and retry backoff. */ compactions: readonly CompactionStatus[]; inbox: readonly QueuedItem[]; @@ -63,18 +66,20 @@ export type AgentEvent = /** `usage` is the partial's current usage, as in the coding agent's JSON mode. */ | { type: "message_update"; usage: Usage; changes: readonly MessageChange[] } | { type: "message_end"; entry: EntryRecord } - | { type: "tool_execution_start"; toolCallId: string; toolName: string; args: JsonObject } - | { + | ({ type: "tool_execution_start"; args: JsonObject } & ToolEventCall) + | ({ type: "tool_execution_update"; - toolCallId: string; - toolName: string; /** A front trim and then an append of the retained window, or its replacement. */ output?: { trimStart?: number; append?: string } | { set: string }; details?: JsonValue; diagnostics?: readonly ToolDiagnostic[]; - } - /** `entry` is absent when the tool task faulted or was orphaned. */ - | { type: "tool_execution_end"; toolCallId: string; toolName: string; entry?: EntryRecord } + } & ToolEventCall) + /** + * A model-issued call ends with its result `entry`, a nested call with its `result`. Both are absent when the tool + * task faulted or was orphaned, or when the call's slot left `pi.live` unfinished: its run ended, or its parent + * settled first. + */ + | ({ type: "tool_execution_end"; entry?: EntryRecord; result?: NestedToolExecutionResult } & ToolEventCall) | { type: "inbox_update"; items: readonly QueuedItem[] } | { type: "submission"; record: SubmissionRecord } | { type: "auto_retry_start"; attempt: number; at: number; errorMessage: string } @@ -88,6 +93,20 @@ export type AgentEvent = /** The task's receipt tells whether it produced a summary; the summary entry has its own events. */ | { type: "compaction_end"; taskId: TaskId; reason: CompactionReason }; +/** + * Which call a tool event is about. `toolCallId` is the call's ID in the transcript and in tool events: the provider's + * ID for a model-issued call, `/` for a nested one. Match events by it; do not parse it. `taskId` + * is the call's tool task, absent for a call that never got one (not offered, or not started in a sequential round). + * A nested call also names the call that made it, by call ID and by tool task. + */ +export type ToolEventCall = { + toolCallId: string; + toolName: string; + taskId?: TaskId; + parentToolCallId?: string; + parentTaskId?: TaskId; +}; + /** Serialized stream of one conversation's event batches, one per commit. */ export interface AgentEventStream { /** The `snapshot` event at attachment. */ @@ -122,6 +141,7 @@ function snapshotOf(view: ConversationView): SnapshotEvent { ...(live.run === undefined ? {} : { run: { inputs: live.run.inputs } }), ...(live.generation === undefined ? {} : { generation: live.generation as SnapshotEvent["generation"] }), tools: live.tools ?? [], + nestedTools: live.nestedTools ?? [], compactions: live.compactions ?? [], inbox: queued(inbox), agent: agent ?? AgentDoc.definition.initial(), @@ -142,9 +162,10 @@ export async function watchEvents( conversationId: ConversationId, context: Context, ): Promise { + const views = conversationViews(harness); let watch!: CommittedWatch; let snapshot!: SnapshotEvent; - await conversationViews(harness).attach( + await views.attach( conversationId, async (initial, release, storage) => { // Generations whose held outcome already ended their turn, read on the line with the snapshot. @@ -154,14 +175,19 @@ export async function watchEvents( let current = initial; snapshot = snapshotOf(initial); // Batches are the watch's values; an overflow delivers a snapshot of the newest view instead. - watch = new CommittedWatch([], release, () => [snapshotOf(current)]); + watch = new CommittedWatch( + [], + release, + (error) => views.report(error), + () => [snapshotOf(current)], + ); return { publication: (before, after, ops, publication, commitContext) => { current = after; const events = translate(conversationId, before, after, ops, publication, held); if (events.length > 0) watch.advance(events, [], commitContext); }, - closeSession: () => watch.closeSession(), + closeSession: (failure) => watch.closeSession(failure), }; }, context, @@ -203,7 +229,18 @@ function translate( const entries: EntryRecord[] = []; const tasks = new Map(); const submissions: SubmissionRecord[] = []; + // Nested results this commit stored, by nested task ID; read now, since the caller's documents may retire next. + const nestedResults = new Map(); for (const change of publication.changes) { + if ( + change.type === "document" && + change.record.kind === NestedResultDoc.definition.kind && + change.record.key !== undefined && + change.value !== null && + change.conversationId === conversationId + ) { + nestedResults.set(change.record.key, change.value.result as unknown as NestedToolExecutionResult); + } if (change.type === "entry" && change.value.conversationId === conversationId) entries.push(change.value); if (change.type === "task" && change.value.conversationId === conversationId) tasks.set(change.value.id, change.value); @@ -221,11 +258,18 @@ function translate( // Progress: tool starts, the in-flight message, tool updates, retry and deferred state. const slotsBefore = new Map((was.live.tools ?? []).map((slot) => [slot.callId, slot])); const slots = now.live.tools ?? []; - for (const slot of slots) { - if (slot.status !== "running" || slotsBefore.get(slot.callId)?.status === "running") continue; + const nestedBefore = new Map((was.live.nestedTools ?? []).map((slot) => [slot.taskId, slot])); + const nested = now.live.nestedTools ?? []; + for (const slot of [...slots, ...nested]) { + const previous = "parentCallId" in slot ? nestedBefore.get(slot.taskId) : slotsBefore.get(slot.callId); + if (slot.status !== "running" || previous?.status === "running") continue; + // A nested slot carries the arguments the call runs with; a model-issued call's are in its intent checkpoint. const checkpoint = slot.taskId === undefined ? undefined : tasks.get(slot.taskId)?.state.checkpoint; - const args = (checkpoint as { arguments?: JsonObject } | undefined)?.arguments ?? {}; - events.push({ type: "tool_execution_start", toolCallId: slot.callId, toolName: slot.name, args }); + const args = + "parentCallId" in slot + ? slot.arguments + : ((checkpoint as { arguments?: JsonObject } | undefined)?.arguments ?? {}); + events.push({ type: "tool_execution_start", ...callOf(slot), args }); } const partialBefore = was.live.generation?.message as AssistantMessage | undefined; const partial = now.live.generation?.message as AssistantMessage | undefined; @@ -236,9 +280,16 @@ function translate( for (const [index, slot] of slots.entries()) { const previous = slotsBefore.get(slot.callId); if (slot.status !== "running" || previous?.status !== "running") continue; - const update = toolUpdate(viewOps, index, slot, previous); + const update = toolUpdate(viewOps, ["tools", index], slot, previous); if (update === undefined) continue; - events.push({ type: "tool_execution_update", toolCallId: slot.callId, toolName: slot.name, ...update }); + events.push({ type: "tool_execution_update", ...callOf(slot), ...update }); + } + for (const [index, slot] of nested.entries()) { + const previous = nestedBefore.get(slot.taskId); + if (slot.status !== "running" || previous?.status !== "running") continue; + const update = toolUpdate(viewOps, ["nestedTools", index], slot, previous); + if (update === undefined) continue; + events.push({ type: "tool_execution_update", ...callOf(slot), ...update }); } const generation = now.live.generation; const generationBefore = was.live.generation; @@ -256,24 +307,34 @@ function translate( // Tools that end in this commit: a slot that becomes done, one created done (a call not offered), or an unfinished // one that vanishes because its run ended. A done slot that vanishes ended earlier. const toolEnds: Extract[] = []; - const endTool = (callId: string, name: string, entryId: number | undefined): void => { + const endTool = (slot: ToolSlot, entryId: number | undefined): void => { const entry = entries.find((candidate) => candidate.id === entryId); - toolEnds.push({ - type: "tool_execution_end", - toolCallId: callId, - toolName: name, - ...(entry === undefined ? {} : { entry }), - }); + toolEnds.push({ type: "tool_execution_end", ...callOf(slot), ...(entry === undefined ? {} : { entry }) }); }; for (const previous of slotsBefore.values()) { if (previous.status === "done") continue; const slot = slots.find((candidate) => candidate.callId === previous.callId); - if (slot?.status === "done") endTool(previous.callId, previous.name, slot.entry); + if (slot?.status === "done") endTool(slot, slot.entry); // A slot whose run ended in this commit may have had its result appended with it, as for unstarted calls. - else if (slot === undefined) endTool(previous.callId, previous.name, resultOf(entries, previous.callId)?.id); + else if (slot === undefined) endTool(previous, resultOf(entries, previous.callId)?.id); } for (const slot of slots) { - if (slot.status === "done" && !slotsBefore.has(slot.callId)) endTool(slot.callId, slot.name, slot.entry); + if (slot.status === "done" && !slotsBefore.has(slot.callId)) endTool(slot, slot.entry); + } + // Nested calls that end in this commit, the same way, children before the calls that made them: the lists hold + // parents first, so walk them backwards. + const endNested = (slot: NestedToolSlot): void => { + const result = nestedResults.get(String(slot.taskId)); + events.push({ type: "tool_execution_end", ...callOf(slot), ...(result === undefined ? {} : { result }) }); + }; + const nestedNow = new Map(nested.map((slot) => [slot.taskId, slot])); + for (const previous of [...nestedBefore.values()].reverse()) { + if (previous.status === "done") continue; + const slot = nestedNow.get(previous.taskId); + if (slot?.status === "done" || slot === undefined) endNested(slot ?? previous); + } + for (const slot of [...nested].reverse()) { + if (slot.status === "done" && !nestedBefore.has(slot.taskId)) endNested(slot); } // Entries in append order; a tool's end directly precedes its result's message, as in the coding agent. @@ -388,14 +449,24 @@ function messageChanges(viewOps: readonly Op[], message: AssistantMessage): Mess return changes; } -/** Output, details, and diagnostics changes of a running slot, from the view operations on it. */ +/** The identifying fields of a slot's tool events. */ +function callOf(slot: ToolSlot | NestedToolSlot): ToolEventCall { + return { + toolCallId: slot.callId, + toolName: slot.name, + ...(slot.taskId === undefined ? {} : { taskId: slot.taskId }), + ...("parentCallId" in slot ? { parentToolCallId: slot.parentCallId, parentTaskId: slot.parentTaskId } : {}), + }; +} + +/** Output, details, and diagnostics changes of a running slot at `at` in `pi.live`, from the view operations on it. */ function toolUpdate( viewOps: readonly Op[], - index: number, - slot: ToolSlot, - previous: ToolSlot, -): Omit, "type" | "toolCallId" | "toolName"> | undefined { - const outputPath = ["docs", "pi.live", "tools", index, "output"]; + at: readonly ["tools" | "nestedTools", number], + slot: ToolSlot | NestedToolSlot, + previous: ToolSlot | NestedToolSlot, +): Omit, "type" | keyof ToolEventCall> | undefined { + const outputPath = ["docs", "pi.live", ...at, "output"]; let trimStart = 0; let append = ""; let set = false; diff --git a/packages/durable/src/harness/generation.ts b/packages/durable/src/harness/generation.ts index ea384605a..85f5b1d0d 100644 --- a/packages/durable/src/harness/generation.ts +++ b/packages/durable/src/harness/generation.ts @@ -273,7 +273,7 @@ export const GenerationTask = defineTask> { - return tx.createTask(ToolTask, { assistant, callId }, { ownership: { kind: "task", taskId: runtime.taskId } }); + return tx.createTask( + ToolTask, + { kind: "model", assistant, callId }, + { ownership: { kind: "task", taskId: runtime.taskId } }, + ); } /** @@ -570,7 +574,7 @@ async function startToolRound( for (const call of calls) { if (!offered.has(call.name)) { const unavailable = harnessError("tool_unavailable", `Tool ${call.name} is not available`); - const result = await appendToolResult(tx, conversationId, call, unavailable, runtime.now()); + const result = await appendToolResult(tx, conversationId, call, unavailable, { timestamp: runtime.now() }); slots.push({ callId: call.id, name: call.name, status: "done", entry: result.id }); continue; } @@ -605,7 +609,11 @@ async function finishToolRound( const outcomes = await runtime.outcomes(tools, context); tools.forEach((id, index) => { const outcome = outcomes[index]!; - controls.set(id, outcome.status === "completed" ? outcome.result.control : undefined); + // Version 1 results have no kind; every result a generation's round reads is a model-issued call's. + controls.set( + id, + outcome.status === "completed" && outcome.result.kind !== "nested" ? outcome.result.control : undefined, + ); }); const slots = (await runtime.snapshot(LiveDoc, conversationId, context))?.tools ?? []; const results = slots.flatMap((slot) => (slot.entry === undefined ? [] : [slot.entry])); @@ -639,6 +647,7 @@ async function finishToolRound( if (users.length > 0) await startRun(tx, conversationId, live, users); } else { delete live.tools; + delete live.nestedTools; if (live.run?.taskId === runtime.taskId) live.run.inputs.push(...users); handOver(live, runtime.taskId, await createGeneration(tx, conversationId)); } diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index 320c712bf..22c0c8d08 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -2,7 +2,7 @@ import type { AttachedReplicatedState, Context, JsonValue } from "@earendil-work import { withAbortSignal, withoutAbortSignal } from "@earendil-works/chord/context"; import { ResetEntry } from "../entries.ts"; import type { ExecutionEnv } from "../env/index.ts"; -import { SessionImpl } from "../session/session.ts"; +import { contained, SessionImpl } from "../session/session.ts"; import type { Transaction } from "../session/transaction.ts"; import type { ConversationId, @@ -164,21 +164,21 @@ class ConversationImpl implements Conversation { /** Session kernel extended with conversation handles and a registry. */ class HarnessImpl extends SessionImpl implements HarnessType { - readonly #storage: Storage; readonly #options: HarnessOptions; readonly #report: (error: unknown) => void; readonly #host: ConversationHost; readonly #tasks: TaskScheduler; readonly #submissions: Submissions; readonly #taskGraph: TaskGraphView; - #closed = false; - constructor(storage: Storage, options: HarnessOptions, context: Context) { - super(storage, options.now); - this.#storage = storage; + constructor(rawStorage: Storage, options: HarnessOptions, context: Context) { + const report = safeReport(options.onReport); + const now = safeNow(options.now, report); + super(rawStorage, now); + // Every component reads and writes through the Session's failure guard: `rawStorage` serves only `super()`. + const storage = this.storage; this.#options = options; - this.#report = options.onReport ?? (() => {}); - const now = options.now ?? Date.now; + this.#report = report; const settings = () => resolveSettings(options.settings); this.#tasks = new TaskScheduler({ session: this, @@ -236,12 +236,12 @@ class HarnessImpl extends SessionImpl implements } resume(): void { - this.#assertOpen(); + this.assertUsable(); this.#tasks.resume(); } getTask(id: TaskId, context: Context): Promise | undefined> { - return this.readOnLine(() => this.#storage.task(id, context)) as Promise< + return this.readOnLine(() => this.storage.task(id, context)) as Promise< TaskRecord | undefined >; } @@ -250,7 +250,7 @@ class HarnessImpl extends SessionImpl implements return this.readOnLine(async () => { const { scheduling, tasks } = await this.#tasks.inspect(this.#options.registry.snapshot()); const scan = (status: "queued" | "placed") => - scanAll((cursor) => this.#storage.scanSubmissions({ status }, SCAN_PAGE_SIZE, cursor, context)); + scanAll((cursor) => this.storage.scanSubmissions({ status }, SCAN_PAGE_SIZE, cursor, context)); const submissions = [...(await scan("queued")), ...(await scan("placed"))].sort((a, b) => a.id - b.id); return { scheduling, tasks, submissions }; }); @@ -285,7 +285,7 @@ class HarnessImpl extends SessionImpl implements /** Sum every conversation's committed `pi.usage`. Each document is read at its own point; totals only grow. */ async usage(context: Context): Promise { const conversations = await this.readOnLine(() => - scanAll((cursor) => this.#storage.scanConversations({}, SCAN_PAGE_SIZE, cursor, context)), + scanAll((cursor) => this.storage.scanConversations({}, SCAN_PAGE_SIZE, cursor, context)), ); const total = UsageDoc.definition.initial(); for (const { id } of conversations) { @@ -311,8 +311,8 @@ class HarnessImpl extends SessionImpl implements } async conversation(id: ConversationId, context: Context): Promise { - this.#assertOpen(); - const record = await this.readOnLine(() => this.#storage.conversation(id, context)); + this.assertUsable(); + const record = await this.readOnLine(() => this.storage.conversation(id, context)); return record === undefined ? undefined : new ConversationImpl(record.id, this.#host); } @@ -320,18 +320,17 @@ class HarnessImpl extends SessionImpl implements return this.#create({ kind: "independent", ownership: options.ownership }, options, context); } - override close(context: Context): Promise { - this.#closed = true; - return super.close(context); + override report(error: unknown): void { + this.#report(error); } - /** Join task invocations after admission is sealed and before Storage closes; writes no task outcome. */ + /** Signal and join task invocations after the close listeners and before Storage closes; writes no task outcome. */ protected override beforeClose(): Promise { return this.#tasks.join(); } async #create(target: CreateTarget, options: CreateOptions, context: Context): Promise { - this.#assertOpen(); + this.assertUsable(); const id = await this.commitWith(async (tx) => { if (target.kind === "root" && (await tx.conversation(ROOT_CONVERSATION_ID)) !== undefined) { return ROOT_CONVERSATION_ID; @@ -362,10 +361,36 @@ class HarnessImpl extends SessionImpl implements await createAgent(tx, record); await this.#options.conversationCreated?.(tx, record); } +} - #assertOpen(): void { - if (this.#closed) throw new Error("Harness is closed"); - } +/** `onReport` that cannot fail: it is the last place an error goes, so its own throw or rejection is dropped. */ +function safeReport(onReport: ((error: unknown) => void) | undefined): (error: unknown) => void { + return (error) => + contained( + () => onReport?.(error), + () => {}, + ); +} + +/** + * The host clock, falling back to `Date.now` when it throws: a broken clock must not stop commits. The first throw is + * reported; the clock is read in every task change, so later ones are not. + */ +export function safeNow(now: (() => number) | undefined, report: (error: unknown) => void): () => number { + if (now === undefined) return Date.now; + let reported = false; + return () => { + try { + return now(); + } catch (error) { + // Set first: a report handler that reads the clock again must not report again. + if (!reported) { + reported = true; + report(error); + } + return Date.now(); + } + }; } /** @@ -427,9 +452,8 @@ export const Harness = { await harness.openTasks(context); } catch (error) { // The caller's context may be what failed open: close without it, and rethrow the open error. - await harness - .close(withoutAbortSignal(context)) - .catch((closeError: unknown) => options.onReport?.(closeError)); + // A failing Storage close fails the Session, which reports it. + await harness.close(withoutAbortSignal(context)).catch(() => {}); throw error; } return harness; diff --git a/packages/durable/src/harness/live.ts b/packages/durable/src/harness/live.ts index 0191e0610..db5c5d73b 100644 --- a/packages/durable/src/harness/live.ts +++ b/packages/durable/src/harness/live.ts @@ -1,21 +1,16 @@ import type { Draft, JsonRepresentation, JsonValue } from "@earendil-works/chord"; -import type { AssistantMessage } from "@earendil-works/pi-ai"; +import type { AssistantMessage, Usage } from "@earendil-works/pi-ai"; import { defineDoc } from "../documents.ts"; import type { Transaction } from "../session/transaction.ts"; -import type { EntryId, SubmissionId, SubmissionSettlement, TaskId, TaskRecord, Tx } from "../types.ts"; +import type { EntryId, JsonObject, SubmissionId, SubmissionSettlement, TaskId, TaskRecord, Tx } from "../types.ts"; import { convertPartial } from "./generation.ts"; import type { SchedulerOutcome } from "./scheduler.ts"; import type { CompactionReason, CompactionResult, ToolDiagnostic } from "./types.ts"; -/** Presentation of one tool call of the current round. */ -export type ToolSlot = { +/** What a running tool call publishes, shared by model-issued and nested calls. */ +type SlotProgress = { callId: string; name: string; - /** - * Absent for a call not started yet (sequential round) and for a call its request did not offer, which starts `done` - * with the `entry` generation wrote. - */ - taskId?: TaskId; status: "pending" | "running" | "done"; /** Retained running output and what the bounds dropped. */ output?: string; @@ -25,10 +20,45 @@ export type ToolSlot = { details?: JsonValue; /** Diagnostics recorded through `api.diagnostic()`. */ diagnostics?: ToolDiagnostic[]; +}; + +/** Presentation of one tool call of the current round. */ +export type ToolSlot = SlotProgress & { + /** + * Absent for a call not started yet (sequential round) and for a call its request did not offer, which starts `done` + * with the `entry` generation wrote. + */ + taskId?: TaskId; /** Result entry once done; absent when the tool task faulted or was orphaned. */ entry?: EntryId; }; +/** + * Presentation of one nested call: a call a running tool made through `executeTool()`. `callId` is + * `/`, the call's ID in tool events and unique because keys contain no `/`. The Harness relates + * slots by task ID: `taskId` is the call's tool task, `parentTaskId` the calling one, model-issued or nested. + */ +export type NestedToolSlot = SlotProgress & { + parentCallId: string; + parentTaskId: TaskId; + taskId: TaskId; + /** + * What the call was made with, replaced by the arguments it runs with, after repair, `beforeTool`, and coercion, + * once it starts executing. A model-issued call's arguments are in its assistant entry instead. + */ + arguments: JsonObject; + /** Once done: how the call ended, for a status line. The result itself is in the caller's `NestedResultDoc`. */ + summary?: NestedToolSummary; +}; + +/** How a nested call ended: an error flag, its execution time, its spend, and up to 500 characters of error text. */ +export type NestedToolSummary = { + isError: boolean; + durationMs?: number; + usage?: JsonRepresentation; + error?: string; +}; + /** Presentation of one live compaction task (spec §8.7). */ export type CompactionStatus = { taskId: TaskId; @@ -56,6 +86,11 @@ export type LiveState = { }; /** The current tool round in call order, from the tool-calling answer until the generation's `tools` phase ends it. */ tools?: ToolSlot[]; + /** + * Nested calls of running tool calls, in creation order, so a parent precedes its children and task IDs ascend. A + * call's nested slots are removed when the call settles. + */ + nestedTools?: NestedToolSlot[]; /** Live compaction tasks in task ID order; absent when none. */ compactions?: CompactionStatus[]; }; @@ -70,9 +105,12 @@ export const LiveDoc = defineDoc({ // REMINDER: a complete base whenever nothing runs (spec §8.2): no generation and no running tool slot. That holds // while idle, in the commit handing a generation over to its tool round, and between tools, so the delta chain // spans at most one generation or the overlapping execution of one round's tools. A slot holds output only while - // running, so every base is small. Do not add a delta-count bound; the tool output benchmark checks this rule. + // running, so every base is small; a nested call's result lives in its caller's task documents, never here. Do not + // add a delta-count bound; the tool output benchmark checks this rule. checkpointWhen: (value) => - value.generation === undefined && !(value.tools ?? []).some((slot) => slot.status === "running"), + value.generation === undefined && + !(value.tools ?? []).some((slot) => slot.status === "running") && + !(value.nestedTools ?? []).some((slot) => slot.status === "running"), }); /** Built-in task kinds that can own `pi.live.run`. */ @@ -81,8 +119,8 @@ const TOOL_TASK_KIND = "pi.tool"; const COMPACTION_TASK_KIND = "pi.compaction"; /** - * End the run owned by `taskId`: settle each of its inputs and remove `run`. Always removes `generation` and `tools`, - * whose presentation belongs to the ending run. + * End the run owned by `taskId`: settle each of its inputs and remove `run`. Always removes `generation`, `tools`, and + * `nestedTools`, whose presentation belongs to the ending run. */ export function endRun(tx: Tx, live: Draft, taskId: TaskId, settlement: SubmissionSettlement): void { if (live.run?.taskId === taskId) { @@ -91,6 +129,7 @@ export function endRun(tx: Tx, live: Draft, taskId: TaskId, settlemen } delete live.generation; delete live.tools; + delete live.nestedTools; } /** Add the status of a compaction task created in this commit; statuses stay in task ID order. */ @@ -113,20 +152,51 @@ export function removeCompactionStatus(live: Draft, taskId: TaskId): if (statuses.length === 0) delete live.compactions; } -/** The slot of tool task `taskId` in the current round, if the round still lists it. */ -export function toolSlot(live: Draft, taskId: TaskId): Draft | undefined { - return live.tools?.find((slot) => slot.taskId === taskId); +/** + * The slot of tool task `taskId`, model-issued or nested, if `pi.live` still lists it. Nested slots are found by binary + * search: a tool may make thousands of nested calls, each of which looks up its slot in several commits, and every + * element a search reads costs a node in the draft's change tracker. Their task IDs ascend: each slot is appended in + * the commit that creates its task, task IDs ascend in commit order, and removal keeps the order. + */ +export function toolSlot(live: Draft, taskId: TaskId): Draft | undefined { + const slot = live.tools?.find((each) => each.taskId === taskId); + if (slot !== undefined || live.nestedTools === undefined) return slot; + const nested = live.nestedTools; + let low = 0; + let high = nested.length - 1; + while (low <= high) { + const middle = (low + high) >>> 1; + const candidate = nested[middle]!; + if (candidate.taskId === taskId) return candidate; + if (candidate.taskId < taskId) low = middle + 1; + else high = middle - 1; + } + return undefined; } -/** Mark a slot done: the result entry, if any, now carries its running output, details, and diagnostics. */ -export function finishSlot(slot: Draft, entry: EntryId | undefined): void { +/** + * Mark a slot done and clear its running output; the caller sets a model-issued call's result `entry`, or a nested + * call's `summary`. Neither is set when the task faulted or was orphaned. + */ +export function finishSlot(slot: Draft): void { slot.status = "done"; - if (entry !== undefined) slot.entry = entry; clearProgress(slot); } +/** Remove the nested slots below tool task `taskId`, transitively; the list holds parents before their children. */ +export function removeNestedSlots(live: Draft, taskId: TaskId): void { + const nested = live.nestedTools; + if (nested === undefined) return; + const removed = new Set([taskId]); + for (const slot of nested) if (removed.has(slot.parentTaskId)) removed.add(slot.taskId); + for (let index = nested.length - 1; index >= 0; index--) { + if (removed.has(nested[index]!.parentTaskId)) nested.splice(index, 1); + } + if (nested.length === 0) delete live.nestedTools; +} + /** Remove what a tool published while running; its result entry or a rerun replaces it. */ -export function clearProgress(slot: Draft): void { +export function clearProgress(slot: Draft): void { delete slot.output; delete slot.droppedBytes; delete slot.droppedLines; @@ -152,8 +222,10 @@ export async function settleSchedulerOutcome( outcome: SchedulerOutcome, ): Promise { if (record.kind === TOOL_TASK_KIND) { - const slot = toolSlot(await tx.doc(LiveDoc, record.conversationId), record.id); - if (slot !== undefined) finishSlot(slot, undefined); + const live = await tx.doc(LiveDoc, record.conversationId); + const slot = toolSlot(live, record.id); + if (slot !== undefined) finishSlot(slot); + removeNestedSlots(live, record.id); return; } if (record.kind === COMPACTION_TASK_KIND) { diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index dcc611f52..bd0bfb481 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -2,6 +2,7 @@ import { type Context, copyJson, type JsonValue } from "@earendil-works/chord"; import { awaitWithContext, withAbortSignal } from "@earendil-works/chord/context"; import type { Models } from "@earendil-works/pi-ai"; import type { ExecutionEnv } from "../env/index.ts"; +import { errorMessage } from "../errors.ts"; import type { SessionImpl } from "../session/session.ts"; import type { Transaction } from "../session/transaction.ts"; import type { @@ -211,6 +212,11 @@ export class TaskScheduler { * one of their tasks fails. */ readonly #failFastChecks = new Set(); + /** + * Live tasks with `abandonOnRestart` found at open, from an earlier Harness. The first reservation pass abort-marks + * them with reason `restart`, so nothing of them or below them runs a phase again. + */ + readonly #abandoned = new Set(); #reconcileScheduled = false; #cascadePending = false; #unsubscribeRegistry: () => void = () => {}; @@ -237,7 +243,7 @@ export class TaskScheduler { /** Load live tasks and change surviving `running` tasks back to `pending`. Dispatches nothing. */ async open(context: Context): Promise { - this.#session.subscribeCommits((publication) => this.#observe(publication)); + this.#session.observeCommits((publication) => this.#observe(publication)); this.#session.subscribeClose(() => this.#seal()); this.#unsubscribeRegistry = this.#registry.subscribe(() => this.#kick()); await this.#session.commitWith(async (tx) => { @@ -249,6 +255,7 @@ export class TaskScheduler { for (const records of scans) { for (const record of records) { this.#live.set(record.id, record); + if (record.abandonOnRestart === true) this.#abandoned.add(record.id); if (record.state.status === "running") { tx.setTask(withState(record, { status: "pending", checkpoint: record.state.checkpoint })); } @@ -269,22 +276,30 @@ export class TaskScheduler { this.#kick(); } - /** Wait for every invocation signalled by `#seal()`. Writes nothing. */ + /** + * Signal every invocation and wait for them; writes nothing. Close calls this after every close listener has run, so + * waits, watches, and streams a task holds end with the Session's reason, not as cancelled by its signal. + */ async join(): Promise { + for (const invocation of this.#invocations.values()) invocation.controller.abort(); await Promise.allSettled([...this.#invocations.values()].map((invocation) => invocation.done)); } /** * Commit the abort mark, or settle a task that no registered definition can take as `orphaned` when nothing it owns * is live, then join the run invocation seen on the line; the commit listener signalled it. The abort invocation - * starts once the task's ordinary owned work is gone. A `completing` task is only marked. + * starts once the task's ordinary owned work is gone. A `completing` task is only marked. A request replaces a + * `restart` mark, so a task abandoned after a restart that waits for its definition is orphaned; with `keepRestart`, + * as for an owner's own cleanup, such a task keeps its mark and waits on. */ - async abort(id: TaskId, context: Context): Promise<"marked" | "terminal"> { + async abort(id: TaskId, context: Context, keepRestart = false): Promise<"marked" | "terminal"> { const marked = await this.#session.commitWith(async (tx) => { const current = await tx.task(id); if (current === undefined) throw new Error(`Task ${id} does not exist`); if (current.state.status === "terminal") return { result: "terminal" as const }; const invocation = this.#invocations.get(id); + // A restart-marked task has no run invocation; it only waits for its abort. + if (keepRestart && current.abortReason === "restart") return { result: "marked" as const }; if (invocation === undefined && current.state.status !== "completing") { await this.#loadScopes(false); if (!this.#ownedLive().has(id)) { @@ -295,18 +310,21 @@ export class TaskScheduler { } } } - if (!current.abortRequested) tx.setTask({ ...current, abortRequested: true }); + if (!current.abortRequested || current.abortReason !== undefined) tx.setTask(withAbortMark(current)); return { result: "marked" as const, run: invocation?.mode === "run" ? invocation : undefined }; }, context); - // The commit listener signalled the run; join it. - if (marked.run !== undefined) await awaitWithContext(marked.run.done, context); + // The commit listener signalled the run; join it. A run that ignores its signal can outlive a failed Session, + // which ends the wait instead. + if (marked.run !== undefined) { + await awaitWithContext(Promise.race([marked.run.done, this.#session.failed]), context); + } return marked.result; } async waitForTask(id: TaskId, context: Context): Promise> { // Check and register on the line so no terminal publication falls between them. const found = await this.#session.readOnLine(async () => { - if (this.#closing) throw closedError(); + if (this.#closing) throw closedError(this.#session); if (this.#live.has(id)) return { promise: this.#taskWaiters.add(id, context) }; const record = await this.#storage.task(id, context); if (record === undefined) throw new Error(`Task ${id} does not exist`); @@ -320,7 +338,7 @@ export class TaskScheduler { * non-background task. */ waitForIdle(conversationId: ConversationId | undefined, context: Context): Promise { - if (this.#closing) return Promise.reject(closedError()); + if (this.#closing) return Promise.reject(closedError(this.#session)); if (this.#idle(conversationId)) return Promise.resolve(); this.#scheduleReconcile(); return this.#idleWaiters.add(conversationId, context); @@ -340,7 +358,7 @@ export class TaskScheduler { if (record.background && !background) continue; if (this.#inScope(parentOf(record), scope, background) !== true) continue; reached.push(record.id); - if (!record.abortRequested) tx.setTask({ ...record, abortRequested: true }); + if (!record.abortRequested || record.abortReason !== undefined) tx.setTask(withAbortMark(record)); } for (const id of queued) { if (this.#inScope({ conversation: id }, scope, background) === true) await this.#withdrawInputs(tx, id); @@ -379,6 +397,8 @@ export class TaskScheduler { const invocation = this.#invocations.get(record.id); if (invocation?.mode === "run") invocation.controller.abort(); } + // A request replacing a `restart` mark upgrades the marks of the work below. + if (previous?.abortReason !== undefined && record.abortReason === undefined) this.#cascadePending = true; const status = record.state.status; if (status === "completing" && previous?.state.status !== "completing") { if (cancellationIntent(record)) this.#cascadePending = true; @@ -419,7 +439,7 @@ export class TaskScheduler { this.#cascadePending = true; } } - // Also retries, with the next commit of any kind, a cascade whose commit failed. + // A cascade that a reservation or step found pending runs with the next commit of any kind. if (this.#cascadePending) this.#scheduleReconcile(); if (!changed) return; this.#settleIdle(); @@ -517,15 +537,23 @@ export class TaskScheduler { await this.#session.commitWith(async (tx) => { if (this.#closing) return; const queued = await this.#loadScopes(cascade); - const marked = new Set(); - const mark = (record: AnyTaskRecord): void => { - if (record.abortRequested || marked.has(record.id)) return; - marked.add(record.id); - tx.setTask({ ...record, abortRequested: true }); + // A cascade from an abandoned owner passes its `restart` reason on; any other intent marks, or upgrades a + // `restart` mark, as a request. Collected first, so a request wins over a `restart` mark of the same pass. + const marks = new Map(); + const mark = (record: AnyTaskRecord, reason: "restart" | undefined): void => { + const staged = marks.get(record.id); + if (staged !== undefined && staged.reason === undefined) return; + if (record.abortRequested && (record.abortReason === undefined || reason !== undefined)) return; + marks.set(record.id, { record, reason }); }; // Loading edges can reveal a cancelled owner, so marks are derived on every pass. for (const record of this.#live.values()) { - if (!record.background && this.#belowCancelled(parentOf(record))) mark(record); + if (record.background) continue; + const owner = this.#cancellingOwner(parentOf(record)); + if (owner === undefined) continue; + // Only an owner abandoned after a restart, and nothing else, passes its reason on. + const restart = owner.abortReason === "restart" && !failedOutcome(owner); + mark(record, restart ? "restart" : undefined); } for (const id of checks) { const waiter = this.#live.get(id); @@ -533,17 +561,17 @@ export class TaskScheduler { // Every other live task: the failed one keeps its own outcome. for (const member of waiter.state.on) { const record = this.#live.get(member); - if (record !== undefined && !failedOutcome(record)) mark(record); + if (record !== undefined && !failedOutcome(record)) mark(record, undefined); } } + for (const { record, reason } of marks.values()) tx.setTask(withAbortMark(record, reason)); for (const id of queued) if (this.#belowCancelled({ conversation: id })) await this.#withdrawInputs(tx, id); await this.#finalize(tx); }, this.#context); } catch (error) { - // Any pass may have staged marks, so a failed one is retried with the next commit. - this.#cascadePending = true; - for (const id of checks) this.#failFastChecks.add(id); - if (!this.#closing) this.#report(error); + // No extension code runs in this commit: a throw is a storage failure, a host callback, or a bug. None is fixed by + // running the pass again, so it fails the Session, which reports it. + this.#failSession(error); } this.#settleIdle(); } @@ -721,27 +749,40 @@ export class TaskScheduler { * background owner without it. Terminal owners never cascade (spec §5.4). */ #belowCancelled(start: Up): boolean { - for (const step of this.#above(start)) { - if ("unknown" in step) return false; - if (!("task" in step)) continue; - const live = this.#live.get(step.task); - if (live !== undefined && cancellationIntent(live)) return true; - if (step.node.background) return false; - } - return false; + return this.#cancellingOwner(start) !== undefined; } - /** Close listener: runs synchronously once admission is sealed, before `join()`. */ + /** The nearest live owner above `start` whose cancellation intent reaches it, if any; see `#belowCancelled`. */ + #cancellingOwner(start: Up): AnyTaskRecord | undefined { + for (const step of this.#above(start)) { + if ("unknown" in step) return undefined; + if (!("task" in step)) continue; + const live = this.#live.get(step.task); + if (live !== undefined && cancellationIntent(live)) return live; + if (step.node.background) return undefined; + } + return undefined; + } + + /** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */ + #failSession(error: unknown): void { + if (!this.#closing) this.#session.fail(error); + } + + /** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */ #seal(): void { this.#closing = true; - this.#unsubscribeRegistry(); - const error = closedError(); + try { + this.#unsubscribeRegistry(); + } catch (error) { + this.#report(error); + } + const error = closedError(this.#session); this.#taskWaiters.rejectAll(error); this.#idleWaiters.rejectAll(error); this.#contexts.clear(); if (this.#expiry !== undefined) clearTimeout(this.#expiry.timer); this.#expiry = undefined; - for (const invocation of this.#invocations.values()) invocation.controller.abort(); } #kick(): void { @@ -759,20 +800,33 @@ export class TaskScheduler { for (const reservation of await this.#reserve()) this.#start(reservation); } } catch (error) { - if (!this.#closing) this.#report(error); + // As in `#reconcile()`: the reservation commit runs no extension code. + this.#failSession(error); } finally { this.#draining = false; - // A wakeup that arrived during a failed pass still needs its pass. + // A wakeup that arrived after the loop's last check still needs its pass. if (this.#dirty) this.#kick(); } } - /** Reserve every eligible task in one commit; orphan abort-marked tasks no definition can take. */ + /** + * Reserve every eligible task in one commit; orphan abort-marked tasks no definition can take, unless abandoned after + * a restart. The first pass after open only abort-marks the abandoned tasks, so later passes see the marks. + */ async #reserve(): Promise { const reservations: Reservation[] = []; + let abandoned: TaskId[] = []; try { await this.#session.commitWith(async (tx) => { if (!this.#enabled || this.#closing) return; + for (const id of this.#abandoned) { + const record = this.#live.get(id); + if (record !== undefined && !record.abortRequested) tx.setTask(withAbortMark(record, "restart")); + } + if (this.#abandoned.size > 0) { + abandoned = [...this.#abandoned]; + return; + } await this.#loadScopes(false); const owned = this.#ownedLive(); // Taken once per pass, and only when some task is a candidate. @@ -782,10 +836,18 @@ export class TaskScheduler { if (record.state.status === "completing") continue; const runnable = record as RunnableTaskRecord; const mode = record.abortRequested ? "abort" : "run"; + // Work below an owner with cancellation intent waits for its cascade mark instead of running a phase; schedule + // the cascade, which may not have run yet. + if (mode === "run" && !record.background && this.#belowCancelled(parentOf(record))) { + this.#cascadePending = true; + this.#scheduleReconcile(); + continue; + } snapshot ??= this.#registry.snapshot(); const resolution = this.#resolve(runnable, snapshot); if (resolution.kind === "blocked") { - if (mode === "abort") { + // An abandoned task waits for its definition, so its abort handler can clean up. + if (mode === "abort" && record.abortReason === undefined) { await this.#terminate(tx, record, { status: "orphaned", reason: resolution.reason }); } continue; @@ -810,6 +872,9 @@ export class TaskScheduler { } throw error; } + // Marked, or already marked or gone: done with them. Reserve again, now seeing the marks. + for (const id of abandoned) this.#abandoned.delete(id); + if (abandoned.length > 0) this.#dirty = true; return reservations; } @@ -956,6 +1021,12 @@ export class TaskScheduler { ): Decision { // 3. abort mark: end; a fresh abort invocation starts once the task's ordinary owned work is gone. if (current.abortRequested) return false; + // An owner's cancellation intent ends it too, before its cascade marks it; reservation then holds it back. + if (!current.background && this.#belowCancelled(parentOf(current))) { + this.#cascadePending = true; + this.#scheduleReconcile(); + return false; + } if (previous === undefined) return true; // 4. uncaught error. if (previous.failure !== undefined) return { fault: previous.failure.error }; @@ -1018,14 +1089,16 @@ export class TaskScheduler { if (decision === true) return current; this.#end(invocation); if (decision !== false) { - const message = decision.fault instanceof Error ? decision.fault.message : String(decision.fault); + const message = errorMessage(decision.fault); await this.#terminate(tx, current!, { status: "faulted", error: { message } }); } return undefined; }, this.#context); } catch (error) { + // The step writes the scheduler's own decision, a fault or a terminal record; a throw there would otherwise + // leave the task running, to be reserved and run again. this.#end(invocation); - if (!this.#closing) this.#report(error); + this.#failSession(error); return undefined; } } @@ -1201,6 +1274,17 @@ export class TaskScheduler { this.#read(invocation, () => this.waitForTask(id, withAbortSignal(invocation.controller.signal, context)), )) as ErasedRuntime["waitForTask"], + abortOwned: (id, context) => + this.#read(invocation, async () => { + const bound = withAbortSignal(invocation.controller.signal, context); + const record = await this.#session.readOnLine(() => this.#storage.task(id, bound)); + if (record?.owner !== invocation.taskId) { + throw new Error(`Task ${id} is not owned by task ${invocation.taskId}`); + } + if (record.state.status === "terminal") return; + await this.abort(id, bound, true); + await this.waitForTask(id, bound); + }), outcomes: ((ids: readonly TaskId[], context: Context) => this.#read(invocation, () => this.#session.readOnLine(async () => { @@ -1294,7 +1378,7 @@ export class TaskScheduler { return this.#session.commitWith( async (tx) => { if (invocation.ended) throw endedError(invocation); - if (this.#closing) throw closedError(); + if (this.#closing) throw closedError(this.#session); const found = this.#live.get(invocation.taskId); if (found === undefined) throw new Error(`Task ${invocation.taskId} is terminal`); if (found.state.status !== "running") throw new Error(`Task ${invocation.taskId} is ${found.state.status}`); @@ -1337,6 +1421,12 @@ export class TaskScheduler { } } +/** The record with an abort mark: `restart` for an abandonment; without a reason, a request, which replaces `restart`. */ +function withAbortMark(record: AnyTaskRecord, reason?: "restart"): AnyTaskRecord { + const { abortReason: _reason, ...rest } = record; + return { ...rest, abortRequested: true, ...(reason === undefined ? {} : { abortReason: reason }) } as AnyTaskRecord; +} + /** A live owner's durable cancellation intent: its abort mark, or a held outcome other than `completed`. */ function cancellationIntent(record: AnyTaskRecord): boolean { return record.state.status !== "terminal" && (record.abortRequested || failedOutcome(record)); diff --git a/packages/durable/src/harness/submissions.ts b/packages/durable/src/harness/submissions.ts index 496b6cbaa..02edc8a33 100644 --- a/packages/durable/src/harness/submissions.ts +++ b/packages/durable/src/harness/submissions.ts @@ -43,10 +43,10 @@ export class Submissions { this.#now = now; this.#queueModes = queueModes; this.#resume = resume; - session.subscribeCommits((publication) => this.#observe(publication)); + session.observeCommits((publication) => this.#observe(publication)); session.subscribeClose(() => { this.#closed = true; - this.#waiters.rejectAll(closedError()); + this.#waiters.rejectAll(closedError(this.#session)); }); } @@ -80,7 +80,7 @@ export class Submissions { if (record === undefined) throw new Error(`Submission ${id} does not exist`); if (isSettled(record)) return { promise: Promise.resolve(record) }; // Close rejects registered waiters synchronously and may begin during the read. - if (this.#closed) throw closedError(); + if (this.#closed) throw closedError(this.#session); return { promise: this.#waiters.add(id, context) }; }); return found.promise; diff --git a/packages/durable/src/harness/task-graph.ts b/packages/durable/src/harness/task-graph.ts index f2065d5d0..9e79aa61a 100644 --- a/packages/durable/src/harness/task-graph.ts +++ b/packages/durable/src/harness/task-graph.ts @@ -2,7 +2,7 @@ import { type AttachedReplicatedState, type Context, type JsonValue, replicatedS import { withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; import { CommittedStateSource, CommittedWatch } from "../session/observation.ts"; -import type { SessionImpl } from "../session/session.ts"; +import { contained, type SessionImpl } from "../session/session.ts"; import type { CommitPublication, ConversationId, @@ -68,12 +68,17 @@ export class TaskGraphView { constructor(session: SessionImpl, storage: Storage) { this.#session = session; this.#storage = storage; - session.subscribeCommits((publication, context) => { - if (this.#mount !== undefined) advance(this.#mount, publication, context); + session.observeCommits((publication, context) => { + if (this.#mount !== undefined) advance(this.#mount, publication, context, (error) => session.report(error)); }); session.subscribeClose(() => { this.#closed = true; - for (const observer of [...(this.#mount?.observers ?? [])]) observer.closeSession(); + for (const observer of [...(this.#mount?.observers ?? [])]) { + contained( + () => observer.closeSession(this.#session.failure), + (error) => session.report(error), + ); + } this.#mount = undefined; }); } @@ -85,7 +90,7 @@ export class TaskGraphView { context, ); try { - return replicatedState(observer); + return replicatedState(observer, { onError: (error) => this.#session.report(error) }); } catch (error) { detach(); throw error; @@ -95,7 +100,7 @@ export class TaskGraphView { /** A serialized exact-frame watch of the graph; cancelling `context` stops it. */ async watch(context: Context): Promise { const { observer } = await this.#attach( - (value, release) => new CommittedWatch(value, release), + (value, release) => new CommittedWatch(value, release, (error) => this.#session.report(error)), context, ); const signal = context.abortSignal; @@ -120,7 +125,7 @@ export class TaskGraphView { }; const observer = create(mount.value, detach); // Close or cancellation may begin while the mount builds; register nothing then. - if (this.#closed) throw closedError(); + if (this.#closed) throw closedError(this.#session); context.abortSignal?.throwIfAborted(); this.#mount = mount; mount.observers.add(observer); @@ -148,8 +153,16 @@ export class TaskGraphView { } } -/** Derive the mount's operations from one publication, apply them, and hand the revision to every observer. */ -function advance(mount: Mount, publication: CommitPublication, context: Context): void { +/** + * Derive the mount's operations from one publication, apply them, and hand the revision to every observer; one that + * throws is reported through `report`, and the others still get it. + */ +function advance( + mount: Mount, + publication: CommitPublication, + context: Context, + report: (error: unknown) => void, +): void { const ops: Op[] = []; // Nodes this publication set or deleted, over the mount's value. const changed = new Map(); @@ -186,7 +199,8 @@ function advance(mount: Mount, publication: CommitPublication, context: Context) if (ops.length === 0) return; mount.value = applyImmutable(mount.value, ops); const frameContext = withoutAbortSignal(context); - for (const observer of [...mount.observers]) observer.advance(mount.value, ops, frameContext); + for (const observer of [...mount.observers]) + contained(() => observer.advance(mount.value, ops, frameContext), report); } function nodeOf(record: AnyTaskRecord, conversations: readonly ConversationId[]): TaskGraphNode { diff --git a/packages/durable/src/harness/tool.ts b/packages/durable/src/harness/tool.ts index df2dac03a..5c75761e6 100644 --- a/packages/durable/src/harness/tool.ts +++ b/packages/durable/src/harness/tool.ts @@ -1,9 +1,12 @@ -import { type Context, copyJson, type JsonValue } from "@earendil-works/chord"; +import { type Context, copyJson, type Draft, type JsonRepresentation, type JsonValue } from "@earendil-works/chord"; import { awaitWithContext } from "@earendil-works/chord/context"; import { overlap } from "@earendil-works/chord/delta"; import type { ImageContent, TextContent, ToolCall, ToolResultMessage } from "@earendil-works/pi-ai"; import { validateToolArguments } from "@earendil-works/pi-ai/utils/validation"; +import { Compile } from "typebox/compile"; +import { defineDoc, defineDocFamily } from "../documents.ts"; import { AssistantEntry, ToolResultEntry } from "../entries.ts"; +import { errorMessage } from "../errors.ts"; import { defineTask } from "../tasks.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, utf8ByteLength } from "../truncate.ts"; import type { @@ -13,55 +16,115 @@ import type { Task, TaskId, TaskOptions, + TaskOutcome, TaskRuntime, Tx, TypedEntry, } from "../types.ts"; import { assignJson } from "./json.ts"; -import { clearProgress, finishSlot, LiveDoc, type ToolSlot, toolSlot } from "./live.ts"; +import { + clearProgress, + finishSlot, + LiveDoc, + type NestedToolSlot, + type NestedToolSummary, + removeNestedSlots, + type ToolSlot, + toolSlot, +} from "./live.ts"; import { boundOutput, OutputBuffer, type OutputLimits, PROGRESS_BYTES_PER_SECOND, Progress } from "./output.ts"; import type { + NestedToolExecutionResult, ToolControl, ToolDiagnostic, ToolExecutionApi, ToolExecutionResult, + ToolHookCall, ToolHooks, ToolRegistration, } from "./types.ts"; import { recordUsage } from "./usage.ts"; -export type ToolTaskInput = { assistant: EntryId; callId: string }; +/** + * A model-issued call, read from its assistant entry, or a nested call a running tool made through `executeTool()`, + * whose call the input carries because no entry holds it. + */ +export type ToolTaskInput = + | { kind: "model"; assistant: EntryId; callId: string } + | { + kind: "nested"; + parent: TaskId; + parentCallId: string; + key: string; + call: ToolCall; + /** `false`: commit no running output, details, or diagnostics to the slot; the result still gets them. */ + progress?: false; + }; export type ToolTaskCheckpoint = | { phase: "call" } /** Durable intent: the final arguments and the replay policy recorded before execution. */ | { phase: "execute"; arguments: JsonObject; replay: "safe" | "unsafe" }; -export type ToolTaskResult = { entryId: EntryId; control?: ToolControl }; +/** + * A model-issued call's result is its transcript entry; a nested call's result is in its caller's + * `NestedResultDoc`, so the receipt stays small and the result retires with the caller. A version 1 task that was + * already holding its outcome as `completing` when the Harness upgraded finishes with a version 1 result, + * `{ entryId, control? }` without `kind`; treat a result without `kind` as a model-issued call's. + */ +export type ToolTaskResult = { kind: "model"; entryId: EntryId; control?: ToolControl } | { kind: "nested" }; + +/** Index of a call's nested calls by key, so a rerun of the call finds the nested calls it already made. */ +const NestedCallsDoc = defineDoc<{ calls: Record> }>({ + kind: "pi.tool.nested", + version: 1, + scope: "task", + initial: () => ({ calls: {} }), +}); + +/** + * The result of one nested call, exactly as `executeTool()` returns it, a member of its caller's family keyed by the + * nested task ID. Written in the nested call's terminal commit; one document per result, so no write rewrites other + * results. Retires with the caller. + */ +export const NestedResultDoc = defineDocFamily<{ result: JsonRepresentation }, JsonObject>({ + kind: "pi.tool.nested-result", + version: 1, + scope: "task", + family: true, + initial: (seed) => seed as { result: JsonRepresentation }, +}); type Runtime = TaskRuntime; type Content = (TextContent | ImageContent)[]; /** * Built-in tool task: resolves the called tool among its phase agent's tools, validates, runs `beforeTool`, records intent, - * executes, runs `afterTool`, and appends the result, all in one `call` handler so nothing separates resolution from - * settlement. `execute` is reached only by recovery and applies the replay rule. + * executes, runs `afterTool`, and settles the result, all in one `call` handler so nothing separates resolution from + * settlement. `execute` is reached only by recovery and applies the replay rule. A model-issued call settles by + * appending its result entry; a nested call by writing its result to its caller's `NestedResultDoc`. */ export const ToolTask = defineTask({ name: "pi.tool", - version: 1, + version: 2, initial: () => ({ phase: "call" }), + // Version 1 had only model-issued calls. + migrate: (input, checkpoint) => ({ + input: { kind: "model", ...(input as { assistant: EntryId; callId: string }) }, + checkpoint: checkpoint as ToolTaskCheckpoint, + }), phases: { call: async (task, runtime, context) => { const call = await readCall(runtime, task.input, context); - const tool = (await runtime.agent(context)).tools.find((each) => each.name === call.name); + const tool = await resolveTool(runtime, task.input, call.name, context); if (tool === undefined) { const error = harnessError("tool_unavailable", `Tool ${call.name} is not available`); - return settle(runtime, call, COMPLETED, () => error, context); + return settle(runtime, task.input, call, COMPLETED, () => error, context); } const prepared = prepare(tool, call.arguments as JsonObject); const checked = "error" in prepared ? prepared : validate(tool, call, prepared.args); - if ("error" in checked) return settle(runtime, call, COMPLETED, () => invalid(checked.error), context); + if ("error" in checked) + return settle(runtime, task.input, call, COMPLETED, () => invalid(checked.error), context); let args = checked.args; let block: string | undefined; await runtime.hooks.each("beforeTool", async (hook) => { @@ -72,29 +135,35 @@ export const ToolTask = defineTask blocked, context); + return settle(runtime, task.input, call, COMPLETED, () => blocked, context); } const validated = validate(tool, call, args); - if ("error" in validated) return settle(runtime, call, COMPLETED, () => invalid(validated.error), context); - const final = validated.args; + if ("error" in validated) + return settle(runtime, task.input, call, COMPLETED, () => invalid(validated.error), context); + // Strict JSON: repair may set optional properties to undefined, which the slot and checkpoint cannot hold. + const final = copyJson(validated.args, { omitUndefinedProperties: true }) as JsonObject; await runtime.commit(async (tx) => { const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); if (slot !== undefined) slot.status = "running"; + // A nested slot shows the arguments the call runs with, which repair, hooks, and coercion may have changed. + // Compared with the input, which the slot got at admission: reading the draft would track every leaf. + const changed = task.input.kind === "nested" && !jsonEqual(task.input.call.arguments as JsonValue, final); + if (changed && slot !== undefined && "parentTaskId" in slot) slot.arguments = final; const intent = { phase: "execute", arguments: final, replay: tool.replay ?? "unsafe" } as const; return { status: "running", checkpoint: intent }; }, context); - await run(runtime, call, tool, final, context); + await run(runtime, task.input, call, tool, final, context); }, /** Recovery after intent: rerun only when the stored and the current policy both say `safe`. */ execute: async (task, runtime, context) => { const { arguments: args, replay } = task.state.checkpoint; const call = await readCall(runtime, task.input, context); - const tool = (await runtime.agent(context)).tools.find((each) => each.name === call.name); + const tool = await resolveTool(runtime, task.input, call.name, context); if (replay === "safe" && tool?.replay === "safe") { // The rerun reports from scratch; clear what the interrupted attempt published. await runtime.commit(async (tx) => { @@ -102,23 +171,42 @@ export const ToolTask = defineTask fromSlot(slot, "interrupted", message), context); + await settle(runtime, task.input, call, ending, (slot) => fromSlot(slot, "interrupted", message), context); }, }, abort: async (task, runtime, context) => { const call = await readCall(runtime, task.input, context); - const message = `Tool ${call.name} was aborted`; - await settle(runtime, call, { status: "aborted" }, (slot) => fromSlot(slot, "aborted", message), context); + // Abandoned after a restart (`TaskOptions.abandonOnRestart`): say whether the call may have run. + const [code, message] = + task.abortReason !== "restart" + ? ["aborted", `Tool ${call.name} was aborted`] + : task.state.checkpoint.phase === "execute" + ? ["interrupted", `Tool ${call.name} was interrupted by a restart and may have partially run`] + : ["abandoned", `Tool ${call.name} was not started: its caller ended with a restart`]; + const ending = { status: "aborted" } as const; + await settle(runtime, task.input, call, ending, (slot) => fromSlot(slot, code, message), context); }, }); -/** The tool call `callId` of the assistant entry. */ -async function readCall(runtime: Runtime, input: ToolTaskInput, context: Context): Promise { +/** The called tool: among the tools the model is offered for a model-issued call, else among those tools may call. */ +async function resolveTool( + runtime: Runtime, + input: ToolTaskInput, + name: string, + context: Context, +): Promise { + const agent = await runtime.agent(context); + return (input.kind === "model" ? agent.tools : agent.callable).find((tool) => tool.name === name); +} + +/** The call: from the input for a nested call, which hooks see with its parent, else from the assistant entry. */ +async function readCall(runtime: Runtime, input: ToolTaskInput, context: Context): Promise { + if (input.kind === "nested") return { ...input.call, parent: { taskId: input.parent, callId: input.parentCallId } }; const entry = await runtime.entry(AssistantEntry, input.assistant, context); const message = entry?.model?.[0]; const call = @@ -140,7 +228,7 @@ function prepare(tool: ToolRegistration, args: JsonObject): Checked { try { return { args: tool.prepareArguments(args) as JsonObject }; } catch (error) { - return { error: errorText(error) }; + return { error: errorMessage(error) }; } } @@ -149,7 +237,7 @@ function validate(tool: ToolRegistration, call: ToolCall, args: JsonObject): Che try { return { args: validateToolArguments(tool, { ...call, arguments: args }) as JsonObject }; } catch (error) { - return { error: errorText(error) }; + return { error: errorMessage(error) }; } } @@ -168,7 +256,8 @@ type Reported = { /** Execute with the resolved implementation, then settle its result. */ async function run( runtime: Runtime, - call: ToolCall, + input: ToolTaskInput, + call: ToolHookCall, tool: ToolRegistration, args: JsonObject, context: Context, @@ -179,7 +268,15 @@ async function run( retain: tool.outputLimits?.retain ?? "head", }; const reported: Reported = { output: new OutputBuffer(limits), limits, diagnostics: [], details: undefined }; - const progress = publishProgress(runtime, reported, context); + const streams = input.kind === "model" || input.progress !== false; + const progress = publishProgress(runtime, reported, streams, context); + /** Nested call admissions this invocation started; cleanup waits for them, so it sees every nested call. */ + const admissions: Promise[] = []; + // A caller that can rerun reattaches to its nested calls and child tasks after a restart; any other caller never + // resumes, so they are abandoned (`TaskOptions.abandonOnRestart`). + const resumes = tool.replay === "safe"; + /** Default keys: the order of this invocation's calls, so a rerun that calls in the same order reattaches. */ + let sequence = 0; let ended = false; const assertLive = (): void => { if (ended) throw new Error(`Tool call ${call.id} has settled`); @@ -204,6 +301,11 @@ async function run( bytesPerSecond: PROGRESS_BYTES_PER_SECOND, } : undefined, + retainedOutput: () => { + assertLive(); + const retained = reported.output.snapshot(); + return { text: retained.text, truncated: retained.droppedBytes > 0 }; + }, diagnostic: (diagnostic) => { assertLive(); reported.diagnostics.push(copyJson(diagnostic, { omitUndefinedProperties: true }) as ToolDiagnostic); @@ -233,9 +335,10 @@ async function run( options: Omit, taskContext: Context, ): Promise> => { + const child = options.ownership.kind === "task" && !resumes ? { abandonOnRestart: true, ...options } : options; let id: TaskId | undefined; await runtime.commit(async (tx) => { - id = await tx.createTask(task, input, options); + id = await tx.createTask(task, input, child); return undefined; }, taskContext); return id!; @@ -246,6 +349,27 @@ async function run( snapshot: runtime.snapshot, snapshotAsOf: runtime.snapshotAsOf, watchDoc: runtime.watchDoc, + executeTool: async (name, toolArgs, callContext, options = {}) => { + assertLive(); + const key = options.key ?? String(++sequence); + if (options.key !== undefined) checkKey(options.key); + const admission = startNestedCall( + runtime, + call.id, + name, + toolArgs, + { key, progress: options.progress, abandonOnRestart: !resumes }, + callContext, + ); + admissions.push(admission); + const id = await admission; + const settled = await runtime.waitForTask(id, callContext); + const stored = await runtime.snapshot(NestedResultDoc, runtime.taskId, String(id), callContext); + // A copy the tool may change; a nested call the scheduler faulted or orphaned stored none. + return stored === undefined + ? fallbackResult(id, name, settled.state.outcome) + : (copyJson(stored.result as unknown as JsonValue) as unknown as NestedToolExecutionResult); + }, }; let result: ToolExecutionResult; @@ -267,18 +391,20 @@ async function run( for (const waiter of await progress.stop()) waiter.reject(error); throw error; } - result = { isError: true, diagnostics: [toolDiagnostic("tool_error", errorText(error))] }; - // A throw, from `execute()` or from building the environment, ends the task `failed`, which cancels what the call owned; it no longer supervises it. The error text - // is already in the result entry. + result = { isError: true, diagnostics: [toolDiagnostic("tool_error", errorMessage(error))] }; + // A throw, from `execute()` or from building the environment, ends the task `failed`, which cancels what the call + // owned; it no longer supervises it. The error text is already in the result. ending = { status: "failed", message: `Tool ${call.name} threw` }; } ended = true; + // No admission starts after `ended`; let those underway commit, so settlement aborts and lists their calls. + await Promise.allSettled(admissions); reported.output.end(); // Details still waiting for a progress commit settle with the terminal commit, the final flush. const pending = await progress.stop(); try { - const settled = await finalResult(runtime, call, result, reported, context); - await settle(runtime, call, ending, () => settled, context, durationMs); + const settled = await finalResult(runtime, input, call, tool, result, reported, context); + await settle(runtime, input, call, ending, () => settled, context, durationMs); } catch (error) { for (const waiter of pending) waiter.reject(error); throw error; @@ -287,13 +413,14 @@ async function run( } /** - * Throttled commits of what the tool reported into its `pi.live.tools` slot, each writing only what changed since the - * last one. + * Throttled commits of what the tool reported into its slot, in `pi.live.tools` or `pi.live.nestedTools`, each writing + * only what changed since the last one; none when `enabled` is false. */ -function publishProgress(runtime: Runtime, reported: Reported, context: Context): Progress { +function publishProgress(runtime: Runtime, reported: Reported, enabled: boolean, context: Context): Progress { let written = { text: "", details: undefined as JsonValue | undefined, diagnostics: 0 }; return new Progress( async () => { + if (!enabled) return 0; // Capture everything synchronously: the tool keeps reporting while the commit is in flight. const snapshot = reported.output.snapshot(); const current = { text: snapshot.text, details: reported.details, diagnostics: reported.diagnostics.length }; @@ -341,71 +468,259 @@ function publishProgress(runtime: Runtime, reported: Reported, context: Context) /** * The settled result: the tool's result with the retained output and last details as fallbacks, its diagnostics after - * those reported through the api, `afterTool` applied, and explicit text bounded, with the Harness's truncation - * diagnostic last. + * those reported through the api, `afterTool` applied, explicit text bounded, and `structuredOutput` checked against the + * tool's schema, with the Harness's diagnostics last. A nested call whose `structuredOutput` breaks the contract gets an + * error result without it; a model-issued call only loses it, and the break is reported to the host. */ async function finalResult( runtime: Runtime, - call: ToolCall, + input: ToolTaskInput, + call: ToolHookCall, + tool: ToolRegistration, result: ToolExecutionResult, reported: Reported, context: Context, ): Promise { const harness: ToolDiagnostic[] = []; - const retained = result.content === undefined ? reported.output.snapshot() : undefined; - const content: Content = - retained === undefined ? result.content! : retained.text === "" ? [] : [{ type: "text", text: retained.text }]; + const retained = result.output === undefined ? reported.output.snapshot() : undefined; + const output: Content = + retained === undefined ? result.output! : retained.text === "" ? [] : [{ type: "text", text: retained.text }]; let final: ToolExecutionResult = { ...result, - content, + output, details: result.details === undefined ? reported.details : result.details, diagnostics: [...reported.diagnostics, ...(result.diagnostics ?? [])], }; await runtime.hooks.each("afterTool", async (hook) => { final = (await hook(call, final, runtime, context)) ?? final; }); - // The retained output's truncation applies only while afterTool kept that content. - if (retained !== undefined && retained.droppedBytes > 0 && final.content === content) { + // The retained output's truncation applies only while afterTool kept that output. + if (retained !== undefined && retained.droppedBytes > 0 && final.output === output) { harness.push(truncated(retained, reported.limits.retain)); } - const bounded = boundContent(final.content ?? [], reported.limits); + const bounded = boundContent(final.output ?? [], reported.limits); if (bounded.droppedBytes > 0) harness.push(truncated(bounded, reported.limits.retain)); - return { ...final, content: bounded.content, diagnostics: [...(final.diagnostics ?? []), ...harness] }; + const broken = structuredOutputError(tool, final); + if (broken !== undefined) { + const { structuredOutput: _dropped, ...rest } = final; + if (input.kind === "nested") { + final = { ...rest, isError: true }; + harness.push(toolDiagnostic("invalid_structured_output", broken)); + } else { + // The model never sees structured output, so its call stands; the host learns of the broken tool. + final = rest; + runtime.report(new Error(broken)); + } + } + // Without a schema, programs get the output itself; the model reads only `output`, so model-issued calls skip it. + if (input.kind === "nested" && tool.structuredOutputSchema === undefined) { + final = { ...final, structuredOutput: outputValue(bounded.content) as JsonValue }; + } + return { ...final, output: bounded.content, diagnostics: [...(final.diagnostics ?? []), ...harness] }; +} + +const structuredValidators = new WeakMap>(); + +/** Why a result's `structuredOutput` breaks the tool's contract, or `undefined` when it keeps it. */ +function structuredOutputError(tool: ToolRegistration, result: ToolExecutionResult): string | undefined { + const schema = tool.structuredOutputSchema; + const value = result.structuredOutput; + if (schema === undefined) { + return value === undefined + ? undefined + : `Tool ${tool.name} returned structuredOutput but declares no structuredOutputSchema`; + } + if (value === undefined) { + return result.isError === true ? undefined : `Tool ${tool.name} returned no structuredOutput`; + } + let validator = structuredValidators.get(schema); + if (validator === undefined) { + validator = Compile(schema); + structuredValidators.set(schema, validator); + } + if (validator.Check(value)) return undefined; + const [first] = validator.Errors(value); + const path = first?.instancePath.replace(/^\//, "").replace(/\//g, ".") || "root"; + return `Tool ${tool.name} returned structuredOutput that does not match its schema: ${path}: ${first?.message ?? "invalid"}`; } /** - * Commit the tool's terminal state: append its result entry, mark its slot done, and complete or end aborted with the - * entry ID. `build` receives the slot so interruption and abort can report the durable partial output. + * Commit the tool's terminal state and mark its slot done. A model-issued call appends its result entry and ends with + * the entry ID; a nested call writes its result to the caller's `NestedResultDoc`, records its usage, and ends with a + * small receipt. Nested calls this call left running are aborted first, so they report into their slots before the + * slots below the call leave `pi.live`. `build` receives the slot so interruption and abort can report the durable + * partial output. */ async function settle( runtime: Runtime, - call: ToolCall, + input: ToolTaskInput, + call: ToolHookCall, ending: Ending, - build: (slot: Readonly | undefined) => ToolExecutionResult, + build: (slot: Readonly | undefined) => ToolExecutionResult, context: Context, durationMs?: number, ): Promise { + // A crash in between leaves the call in `execute`: a safe rerun reattaches to the calls, an interruption ends here. + const index = await runtime.snapshot(NestedCallsDoc, runtime.taskId, context); + const nestedIds = Object.values(index?.calls ?? {}).sort((a, b) => a - b); + // Mark them all before waiting for any: one may run until a sibling is cancelled. + await Promise.all(nestedIds.map((id) => runtime.abortOwned(id, context))); await runtime.commit(async (tx) => { - const slot = toolSlot(await tx.doc(LiveDoc, runtime.conversationId), runtime.taskId); + const conversationId = runtime.conversationId; + const live = await tx.doc(LiveDoc, conversationId); + const slot = toolSlot(live, runtime.taskId); const result = build(slot); - const entry = await appendToolResult(tx, runtime.conversationId, call, result, runtime.now(), durationMs); - if (slot !== undefined) finishSlot(slot, entry.id); - const entryId = entry.id; - if (ending.status === "aborted") - return { status: "terminal", outcome: { status: "aborted", result: { entryId } } }; + let settled: ToolTaskResult; + if (input.kind === "nested") { + const nested = nestedResult(runtime.taskId, result, durationMs); + if (nested.usage !== undefined) await recordUsage(tx, conversationId, "tools", call.name, nested.usage); + // The caller is never terminal before its owned nested calls; a missing entry is a bug, not a race. + if ((await tx.doc(NestedCallsDoc, input.parent)).calls[input.key] !== runtime.taskId) { + throw new Error(`Nested call ${call.id} is not in its caller's index`); + } + const stored = { result: nested as unknown as JsonRepresentation }; + await tx.doc(NestedResultDoc, input.parent, String(runtime.taskId), stored); + if (slot !== undefined && "parentTaskId" in slot) { + finishSlot(slot); + slot.summary = summaryOf(nested, result.output ?? []); + } + settled = { kind: "nested" }; + } else { + const timing = { timestamp: runtime.now(), durationMs }; + const entry = await appendToolResult(tx, conversationId, call, result, timing); + if (slot !== undefined && !("parentTaskId" in slot)) { + finishSlot(slot); + slot.entry = entry.id; + } + // Tools build control objects freely; drop keys set to undefined so the task result is strict JSON. + const control = + result.control === undefined || ending.status !== "completed" + ? {} + : { control: copyJson(result.control as JsonValue, { omitUndefinedProperties: true }) as ToolControl }; + settled = { kind: "model", entryId: entry.id, ...control }; + } + // A call that made no nested calls has no slots below it; skipping the scan keeps many leaves linear. + if (nestedIds.length > 0) removeNestedSlots(live, runtime.taskId); + if (ending.status === "aborted") return { status: "terminal", outcome: { status: "aborted", result: settled } }; if (ending.status === "failed") { const error = { message: ending.message }; - return { status: "terminal", outcome: { status: "failed", error, result: { entryId } } }; + return { status: "terminal", outcome: { status: "failed", error, result: settled } }; } - // Tools build control objects freely; drop keys set to undefined so the task result is strict JSON. - const control = - result.control === undefined - ? {} - : { control: copyJson(result.control as JsonValue, { omitUndefinedProperties: true }) as ToolControl }; - return { status: "terminal", outcome: { status: "completed", result: { entryId, ...control } } }; + return { status: "terminal", outcome: { status: "completed", result: settled } }; }, context); } +/** + * A nested call's result, as stored and returned: strict JSON, without `output`, which only the model reads, and + * without `control`, which only model-issued calls apply. + */ +function nestedResult( + taskId: TaskId, + result: ToolExecutionResult, + durationMs: number | undefined, +): NestedToolExecutionResult { + const nested = { + taskId, + structuredOutput: result.structuredOutput, + isError: result.isError ?? false, + details: result.details, + diagnostics: result.diagnostics ?? [], + usage: result.usage, + durationMs, + }; + return copyJson(nested as unknown as JsonValue, { + omitUndefinedProperties: true, + }) as unknown as NestedToolExecutionResult; +} + +/** + * Create nested call `key` of the call `parentCallId` in one commit: its tool task, owned by the calling task, its slot, + * and its index entry. A key already in the index returns that call's task when it names the same tool and arguments. + */ +async function startNestedCall( + runtime: Runtime, + parentCallId: string, + name: string, + args: JsonObject, + options: { readonly key: string; readonly progress: boolean | undefined; readonly abandonOnRestart: boolean }, + context: Context, +): Promise> { + const key = options.key; + const call: ToolCall = { + type: "toolCall", + id: `${parentCallId}/${key}`, + name, + arguments: copyJson(args, { omitUndefinedProperties: true }) as JsonObject, + }; + let id: TaskId | undefined; + await runtime.commit(async (tx) => { + const index = await tx.doc(NestedCallsDoc, runtime.taskId); + const existing = Object.hasOwn(index.calls, key) ? index.calls[key] : undefined; + if (existing !== undefined) { + const input = (await tx.task(existing))?.input as ToolTaskInput | undefined; + const same = + input?.kind === "nested" && + input.parent === runtime.taskId && + input.call.name === name && + jsonEqual(input.call.arguments as JsonValue, call.arguments as JsonValue); + if (!same) throw new Error(`Nested call ${call.id} was already made with another tool or other arguments`); + id = existing as TaskId; + return undefined; + } + const input: ToolTaskInput = { + kind: "nested", + parent: runtime.taskId, + parentCallId, + key, + call, + ...(options.progress === false ? { progress: false } : {}), + }; + const created = await tx.createTask(ToolTask, input, { + ownership: { kind: "task", taskId: runtime.taskId }, + abandonOnRestart: options.abandonOnRestart, + }); + index.calls[key] = created; + const live = await tx.doc(LiveDoc, runtime.conversationId); + live.nestedTools ??= []; + const slot: NestedToolSlot = { + callId: call.id, + parentCallId, + parentTaskId: runtime.taskId, + name, + taskId: created, + arguments: call.arguments as JsonObject, + status: "pending", + }; + live.nestedTools.push(slot as Draft); + id = created; + return undefined; + }, context); + return id!; +} + +/** + * What programs get of a tool's output when it declares no schema: one text item as its string, one image as itself, + * nothing as `""`, and anything else as the content list. + */ +function outputValue(output: Content): string | ImageContent | Content { + if (output.length === 0) return ""; + if (output.length > 1) return output; + const [only] = output; + return only!.type === "text" ? only!.text : only!; +} + +/** What the caller gets for a nested call without a stored result: one the scheduler faulted or orphaned. */ +function fallbackResult(taskId: TaskId, name: string, outcome: TaskOutcome): NestedToolExecutionResult { + const message = + outcome.status === "faulted" + ? `Tool ${name} failed: ${outcome.error.message}` + : outcome.status === "orphaned" + ? `Tool ${name} could not resume: ${outcome.reason}` + : `Tool ${name} ended without a result`; + const error = harnessError(outcome.status, message); + return { taskId, isError: true, diagnostics: error.diagnostics ?? [] }; +} + /** * How a tool task ends; the result entry is appended either way. `failed` (execution threw or was interrupted) * records cancellation intent for the conversations the call owns; a result with `isError` still completes. @@ -415,13 +730,17 @@ type Ending = { readonly status: "completed" | "aborted" } | { readonly status: const COMPLETED: Ending = { status: "completed" }; /** An error result from the slot's durable partial output, details, and diagnostics. */ -function fromSlot(slot: Readonly | undefined, code: string, message: string): ToolExecutionResult { +function fromSlot( + slot: Readonly | undefined, + code: string, + message: string, +): ToolExecutionResult { const diagnostics = [...(slot?.diagnostics ?? [])]; const droppedBytes = slot?.droppedBytes ?? 0; if (droppedBytes > 0) diagnostics.push(truncated({ droppedBytes, droppedLines: slot?.droppedLines ?? 0 })); diagnostics.push(toolDiagnostic(code, message)); return { - content: slot?.output === undefined || slot.output === "" ? [] : [{ type: "text", text: slot.output }], + output: slot?.output === undefined || slot.output === "" ? [] : [{ type: "text", text: slot.output }], isError: true, ...(slot?.details === undefined ? {} : { details: slot.details }), diagnostics, @@ -430,7 +749,7 @@ function fromSlot(slot: Readonly | undefined, code: string, message: s /** An error result the Harness writes itself: no content and one `error` diagnostic with `code`. */ export function harnessError(code: string, message: string): ToolExecutionResult { - return { content: [], isError: true, diagnostics: [toolDiagnostic(code, message)] }; + return { output: [], isError: true, diagnostics: [toolDiagnostic(code, message)] }; } function toolDiagnostic(code: string, message: string): ToolDiagnostic { @@ -450,6 +769,27 @@ function truncated( }; } +/** Error text a nested call's summary keeps. */ +const MAX_SUMMARY_ERROR_CHARS = 500; + +/** How a nested call ended, for its slot: error text, from its diagnostics or else its `output`, bounded. */ +function summaryOf(result: NestedToolExecutionResult, output: Content): NestedToolSummary { + const error = result.isError ? errorTextOf(result, output).slice(0, MAX_SUMMARY_ERROR_CHARS) : ""; + return { + isError: result.isError, + ...(result.durationMs === undefined ? {} : { durationMs: result.durationMs }), + ...(result.usage === undefined ? {} : { usage: result.usage }), + ...(error === "" ? {} : { error }), + }; +} + +/** The error messages of a nested result, else the text items of its output, joined with newlines. */ +function errorTextOf(result: NestedToolExecutionResult, output: Content): string { + const errors = result.diagnostics.filter((diagnostic) => diagnostic.severity === "error"); + if (errors.length > 0) return errors.map((diagnostic) => diagnostic.message).join("\n"); + return output.flatMap((item) => (item.type === "text" ? [item.text] : [])).join("\n"); +} + /** * Append a `pi.tool-result` entry. The content ends with the rendered diagnostics, so the stored message is exactly * what the model sees; `data` keeps the structured list. A result's usage is added to `pi.usage` in the same commit. @@ -459,11 +799,11 @@ export async function appendToolResult( conversationId: ConversationId, call: ToolCall, result: ToolExecutionResult, - timestamp: number, - durationMs?: number, + meta: { readonly timestamp: number; readonly durationMs?: number }, ): Promise> { + const { timestamp, durationMs } = meta; const diagnostics = [...(result.diagnostics ?? [])]; - const content: Content = [...(result.content ?? [])]; + const content: Content = [...(result.output ?? [])]; if (diagnostics.length > 0) content.push({ type: "text", text: renderDiagnostics(diagnostics) }); const message = { role: "toolResult", @@ -504,6 +844,29 @@ function boundContent( return { content: result, droppedBytes: bounded.droppedBytes, droppedLines: bounded.droppedLines }; } -function errorText(error: unknown): string { - return error instanceof Error ? error.message : String(error); +/** + * Explicit nested call keys are path segments of the call ID, so IDs of different nested calls never collide, and are + * never plain positive integers, which default keys use. `__proto__` would not land in the index as an own key. + */ +function checkKey(key: string): void { + if (key === "" || key.includes("/") || key === "__proto__" || /^[1-9][0-9]*$/.test(key)) { + throw new Error( + `Nested call key ${JSON.stringify(key)} must be non-empty, without "/", not "__proto__", and not a positive integer`, + ); + } +} + +/** Structural JSON equality; object key order does not matter. */ +function jsonEqual(a: JsonValue, b: JsonValue): boolean { + if (a === b) return true; + if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) return false; + if (Array.isArray(a) || Array.isArray(b)) { + return ( + Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, i) => jsonEqual(item, b[i]!)) + ); + } + const keys = Object.keys(a); + return ( + keys.length === Object.keys(b).length && keys.every((key) => Object.hasOwn(b, key) && jsonEqual(a[key]!, b[key]!)) + ); } diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 9e44ff89c..4d1597331 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -140,9 +140,18 @@ export type ToolDiagnostic = { readonly code?: string; }; +/** + * What a tool returns, on three channels: `output` for the model, `structuredOutput` for programs that call the tool + * (`executeTool()` callers, code mode scripts), and `details` for UIs. + */ export type ToolExecutionResult = { - /** Omitted: the retained `output()` text becomes the content. */ - readonly content?: ToolResultMessage["content"]; + /** For the model. Omitted: the retained `output()` text. */ + readonly output?: ToolResultMessage["content"]; + /** + * For programs. Required unless `isError` when the tool declares `structuredOutputSchema`, and validated against it; + * not allowed without one, where programs get the output itself (`NestedToolExecutionResult.structuredOutput`). + */ + readonly structuredOutput?: JsonValue; readonly isError?: boolean; /** Omitted: the last `details()` value becomes the details. */ readonly details?: TDetails; @@ -153,6 +162,28 @@ export type ToolExecutionResult = { readonly control?: ToolControl; }; +/** + * Result of a nested call, as `executeTool()` returns it. Programs read `structuredOutput`; the output for the model + * is not kept, and `control` does not apply. + */ +export type NestedToolExecutionResult = { + /** The nested call's tool task. */ + readonly taskId: TaskId; + /** + * The value for programs. A tool with `structuredOutputSchema`: its validated `structuredOutput`, absent on an error + * result that has none. A tool without: its bounded output, success or error, as a string for one text item, an + * `ImageContent` for one image, `""` for none, and the content list otherwise. Absent on an error result the Harness + * wrote itself (unavailable, invalid, blocked, interrupted, aborted); `diagnostics` say what went wrong. + */ + readonly structuredOutput?: JsonValue; + readonly isError: boolean; + readonly details?: JsonValue; + readonly diagnostics: readonly ToolDiagnostic[]; + readonly usage?: Usage; + /** Execution time of the attempt that produced the result; absent when it never executed. */ + readonly durationMs?: number; +}; + /** Whether the tools of one round run at once or one after another in call order. */ export type ToolExecutionMode = "parallel" | "sequential"; @@ -176,7 +207,7 @@ export interface ToolExecutionApi extend /** Built by `HarnessOptions.env` for this call; `undefined` without an environment. */ readonly env: ExecutionEnv | undefined; /** - * Append running output; it becomes the result content when the result omits `content`. `skipped` counts output + * Append running output; it becomes the result's `output` when the result omits it. `skipped` counts output * omitted before the chunk, as reported by an environment given `outputWindow` (`ShellOutputInfo.skipped`). */ output(chunk: string | Uint8Array, skipped?: ShellOutputSkip): void; @@ -186,6 +217,11 @@ export interface ToolExecutionApi extend * transforms text must also replace this with `undefined`, so skipped text cannot bypass its transform. */ readonly outputWindow: ShellOutputWindow | undefined; + /** + * The output retained so far, as the model will see it when the result omits `output`, and whether earlier output was + * dropped; for a tool that also returns that text in its `structuredOutput`. + */ + retainedOutput(): { readonly text: string; readonly truncated: boolean }; /** Record a model-visible remark about this call. */ diagnostic(diagnostic: ToolDiagnostic): void; /** Replace running details; the last value becomes the result details when the result omits `details`. */ @@ -193,6 +229,10 @@ export interface ToolExecutionApi extend commit(change: (tx: Tx) => T | Promise, context: Context): Promise; memo(name: string, context: Context): Promise; memo(name: string, candidate: T, context: Context): Promise; + /** + * Create a task in one commit. A child task of a tool that is not replay-safe defaults to `abandonOnRestart`: the + * tool never resumes after a restart, so nothing awaits the child then. + */ createTask( task: Task, input: I, @@ -203,6 +243,39 @@ export interface ToolExecutionApi extend waitForTask(id: TaskId, context: Context): Promise>; /** Invocation-bound handle of an existing conversation, such as one this tool created in `commit()`. */ conversation(id: ConversationId, context: Context): Promise; + /** + * Run tool `name` as a nested call of this one and wait for its result. The nested call is its own `pi.tool` task, + * owned by this call: it resolves the tool among the conversation's tools, validates, runs the `ToolTask` hooks (which + * see `call.parent`), applies output limits and the tool's replay policy, and reports progress in `pi.live.nestedTools`. + * Its result returns here instead of entering the transcript. A call that is blocked, invalid, throws, is interrupted, + * or is aborted returns an `isError` result. + * + * The nested call's key names it within this call; its call ID is `/`. By default the key is the call's + * position among this invocation's nested calls, `1`, `2`, ... An explicit `key` must be non-empty, without `/`, not + * `__proto__`, and not a positive integer, so it never collides with a default one. A rerun of a replay-safe tool + * that makes a call with a used key, by calling in the same order or by passing the same `key`, gets that nested call + * back, finished or still running, instead of starting another. A nested call a crash interrupted before its result + * committed follows its own replay policy, so this is not exactly-once execution. Reusing a key with another tool or + * other arguments throws. A tool that is not replay-safe never reruns: after a restart, its unfinished nested calls + * are abandoned (`TaskOptions.abandonOnRestart`) before they run again. + * + * `progress: false` commits no running output, details, or diagnostics to the nested call's slot, only its status; + * the result still carries its details and diagnostics, and a schema-less tool's output as `structuredOutput`. An + * interrupted or aborted call's result is built from its slot, so it then has none of them. Either way, programs never + * get the partial output of an interrupted or aborted call. + * + * Nested calls run as soon as they are made, whatever their `executionMode`; the caller decides what runs at once by + * what it awaits. Cancelling `context` stops only the wait; the nested call runs on. When this call settles, nested + * calls it left running are aborted first. A nested result's `usage` is already counted under its tool; a result + * that passes it on counts it again. Rejects, instead of returning a result, once `execute()` has returned, when this + * call is aborted, or when the Harness closes. + */ + executeTool( + name: string, + args: JsonObject, + context: Context, + options?: { readonly key?: string; readonly progress?: boolean }, + ): Promise; } /** @@ -223,6 +296,17 @@ export type ToolRegistration< * still validated against `parameters`. */ prepareArguments?(args: unknown): Static; + /** + * Schema of `structuredOutput`, which programs that call the tool receive. Omitted: they receive the result's bounded + * output, success or error, as `NestedToolExecutionResult.structuredOutput` describes. + */ + readonly structuredOutputSchema?: TSchema; + /** + * Who may call the tool: the model, other tools through `executeTool()`, or both (default). A code mode tool is + * `["model"]`, so scripts cannot start scripts; tools behind code mode, such as MCP tools, can be `["tools"]`, so the + * model is never offered them. + */ + readonly callers?: readonly ("model" | "tools")[]; readonly outputLimits?: { readonly maxBytes?: number; readonly maxLines?: number; @@ -310,8 +394,13 @@ export type AgentState = { thinkingLevel?: ModelThinkingLevel; /** An array selects exactly these extensions, in order. An object edits the host default selection. */ extensions?: string[] | { add?: string[]; remove?: string[] }; - /** Filters the selected extensions' tools. An array offers exactly these, in order. */ + /** Filters the selected extensions' tools: the enabled tools. An array enables exactly these, in order. */ tools?: string[] | { remove: string[] }; + /** + * Filters the enabled tools the model may call (`callers` includes `model`): the tools offered to it. An array offers + * exactly these, in order. Never widens: other tools can still call the ones it leaves out. + */ + modelTools?: string[] | { remove: string[] }; /** Rendered after every extension section, as the section `instructions`. */ instructions?: string; /** Directory within the environment's file system, passed to `HarnessOptions.env`. */ @@ -327,6 +416,7 @@ export type AgentChange = { | { readonly add?: readonly Extension[]; readonly remove?: readonly Extension[] } | null; readonly tools?: readonly ToolRegistration[] | { readonly remove: readonly ToolRegistration[] } | null; + readonly modelTools?: readonly ToolRegistration[] | { readonly remove: readonly ToolRegistration[] } | null; readonly instructions?: string | null; readonly cwd?: string | null; }; @@ -336,8 +426,10 @@ export type Agent = { readonly model?: ModelRef; readonly thinkingLevel: ModelThinkingLevel; readonly extensions: readonly Extension[]; - /** The tools a request offers, in order. */ + /** The tools a request offers, in order: enabled, callable by the model, and selected by `modelTools`. */ readonly tools: readonly Tool[]; + /** The tools nested calls (`executeTool()`) resolve among, in order: enabled and callable by tools. */ + readonly callable: readonly Tool[]; /** Extension sections, then `instructions` when set. */ readonly sections: readonly PromptSection[]; readonly instructions?: string; @@ -464,7 +556,10 @@ export type HarnessOptions = { */ readonly conversationCreated?: (tx: Tx, conversation: ConversationRecord) => void | Promise; readonly now?: () => number; - /** Receives extension failures that do not fail the calling operation. Must not throw. */ + /** + * Receives failures that do not fail the calling operation, such as a throwing hook or listener, and the error that + * fails the Session, once. Its own throw or rejection is dropped. + */ readonly onReport?: (error: unknown) => void; }; @@ -641,17 +736,22 @@ export interface GenerationHooks { afterTools(assistant: EntryId, results: readonly EntryId[], api: HookApi, context: Context): void | Promise; } -/** Hooks of the built-in tool task. */ +/** A call as tool hooks see it; `parent` is set for a nested call, one a running tool made through `executeTool()`. */ +export type ToolHookCall = ToolCall & { + readonly parent?: { readonly taskId: TaskId; readonly callId: string }; +}; + +/** Hooks of the built-in tool task, for model-issued and nested calls. */ export interface ToolHooks { /** Before intent; the first `block` wins, otherwise `arguments` replace the call's arguments. A throw blocks. */ beforeTool( - call: ToolCall, + call: ToolHookCall, api: HookApi, context: Context, ): HookResult<{ readonly arguments?: JsonObject; readonly block?: string }>; - /** After execution, before the result entry; replaces the result. */ + /** After execution, before the result is committed; replaces the result. */ afterTool( - call: ToolCall, + call: ToolHookCall, result: ToolExecutionResult, api: HookApi, context: Context, diff --git a/packages/durable/src/harness/util.ts b/packages/durable/src/harness/util.ts index a92b9d94f..0b41372f2 100644 --- a/packages/durable/src/harness/util.ts +++ b/packages/durable/src/harness/util.ts @@ -1,4 +1,5 @@ import type { Context } from "@earendil-works/chord"; +import { SessionFailed } from "../errors.ts"; import type { Cursor, Page } from "../types.ts"; /** Pending waits by key. Each settles once: through `resolve`, `rejectAll`, or cancellation of its context. */ @@ -53,6 +54,8 @@ export async function scanAll(scan: (cursor: Cursor | undefined) => Promise

{ - for (const [id, mount] of this.#mounts) advance(id, mount, publication, context); + session.observeCommits((publication, context) => { + for (const [id, mount] of this.#mounts) + advance(id, mount, publication, context, (error) => this.report(error)); }); session.subscribeClose(() => { this.#closed = true; for (const mount of this.#mounts.values()) - for (const observer of [...mount.observers]) observer.closeSession(); + for (const observer of [...mount.observers]) { + contained( + () => observer.closeSession(this.#session.failure), + (error) => this.report(error), + ); + } this.#mounts.clear(); }); } + /** Report an observer's listener error through the Harness. */ + report(error: unknown): void { + this.#session.report(error); + } + /** A disposable read-only Chord state of the view. */ async state(id: ConversationId, context: Context): Promise> { const { observer, detach } = await this.attach( @@ -104,7 +115,7 @@ export class ConversationViews { context, ); try { - return replicatedState(observer); + return replicatedState(observer, { onError: (error) => this.#session.report(error) }); } catch (error) { detach(); throw error; @@ -115,7 +126,8 @@ export class ConversationViews { async watch(id: ConversationId, context: Context): Promise> { const { observer } = await this.attach( id, - (value, release) => new CommittedWatch(value, release), + (value, release) => + new CommittedWatch(value, release, (error) => this.#session.report(error)), context, ); const signal = context.abortSignal; @@ -145,7 +157,7 @@ export class ConversationViews { }; const observer = await create(mount.value, detach, this.#storage); // Close or cancellation may begin while the mount hydrates; register nothing then. - if (this.#closed) throw closedError(); + if (this.#closed) throw closedError(this.#session); context.abortSignal?.throwIfAborted(); this.#mounts.set(id, mount); mount.observers.add(observer); @@ -170,8 +182,17 @@ export class ConversationViews { } } -/** Derive the mount's operations from one publication, apply them, and hand the revision to every observer. */ -function advance(id: ConversationId, mount: Mount, publication: CommitPublication, context: Context): void { +/** + * Derive the mount's operations from one publication, apply them, and hand the revision to every observer; one that + * throws is reported through `report`, and the others still get it. + */ +function advance( + id: ConversationId, + mount: Mount, + publication: CommitPublication, + context: Context, + report: (error: unknown) => void, +): void { const docOps: Op[] = []; const entryOps: Op[] = []; let entries = mount.value.entries; @@ -215,10 +236,12 @@ function advance(id: ConversationId, mount: Mount, publication: CommitPublicatio if (ops.length > 0) { const value = docOps.length === 0 ? mount.value : applyImmutable(mount.value, docOps); mount.value = entries === value.entries ? value : { ...value, entries }; - for (const observer of [...mount.observers]) observer.advance?.(mount.value, ops, frameContext); + for (const observer of [...mount.observers]) { + contained(() => observer.advance?.(mount.value, ops, frameContext), report); + } } for (const observer of [...mount.observers]) { - observer.publication?.(before, mount.value, ops, publication, frameContext); + contained(() => observer.publication?.(before, mount.value, ops, publication, frameContext), report); } } diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index 2b1dc3037..4a9e205ab 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -8,7 +8,7 @@ export { ToolResultEntry, UserEntry, } from "./entries.ts"; -export { ConversationBusy, ReadAfterWrite, StorageRejected } from "./errors.ts"; +export { ConversationBusy, ReadAfterWrite, SessionFailed, StorageRequestError } from "./errors.ts"; export { AgentDoc, configure, @@ -28,6 +28,7 @@ export { type AgentEventStream, type MessageChange, type SnapshotEvent, + type ToolEventCall, watchEvents, } from "./harness/events.ts"; export { @@ -38,11 +39,24 @@ export { } from "./harness/generation.ts"; export { Harness } from "./harness/harness.ts"; export { InboxDoc, type InboxItem, type InboxState } from "./harness/inbox.ts"; -export { type CompactionStatus, LiveDoc, type LiveState, type ToolSlot } from "./harness/live.ts"; +export { + type CompactionStatus, + LiveDoc, + type LiveState, + type NestedToolSlot, + type NestedToolSummary, + type ToolSlot, +} from "./harness/live.ts"; export { ProviderDoc, type ProviderState } from "./harness/provider.ts"; export { createRegistry } from "./harness/registry.ts"; export type { TaskGraph, TaskGraphNode, TaskGraphState, TaskGraphWatch } from "./harness/task-graph.ts"; -export { ToolTask, type ToolTaskCheckpoint, type ToolTaskInput, type ToolTaskResult } from "./harness/tool.ts"; +export { + NestedResultDoc, + ToolTask, + type ToolTaskCheckpoint, + type ToolTaskInput, + type ToolTaskResult, +} from "./harness/tool.ts"; export type { Agent, AgentChange, @@ -73,6 +87,7 @@ export type { HooksOf, InputSubmissionDraft, ModelRef, + NestedToolExecutionResult, ProgressPolicy, PromptInput, PromptSection, @@ -91,6 +106,7 @@ export type { ToolExecutionApi, ToolExecutionMode, ToolExecutionResult, + ToolHookCall, ToolHooks, ToolRegistration, UserInput, @@ -154,6 +170,7 @@ export type { Session, SessionDocFamilyToken, SessionDocToken, + SessionEnd, Storage, StorageWrite, StoredDocument, diff --git a/packages/durable/src/session/observation.ts b/packages/durable/src/session/observation.ts index ac74d438a..2edfb7465 100644 --- a/packages/durable/src/session/observation.ts +++ b/packages/durable/src/session/observation.ts @@ -7,6 +7,7 @@ import type { } from "@earendil-works/chord"; import { withoutAbortSignal } from "@earendil-works/chord/context"; import type { Op } from "@earendil-works/chord/delta"; +import { errorMessage } from "../errors.ts"; import type { JsonObject, WatchEnd, WatchHandle } from "../types.ts"; export type ObservedDocumentValue = Readonly | null; @@ -59,6 +60,7 @@ export class CommittedStateSource implements Replicat for (const attachment of [...this.#attachments]) attachment.publish(frame); } + /** Stop publishing; the attached states keep their last value. A failure ends them the same way. */ closeSession(): void { if (this.#closed) return; for (const attachment of [...this.#attachments]) attachment.dispose(); @@ -154,30 +156,30 @@ type WatchFrame = { */ export class CommittedWatch implements WatchHandle { readonly #detach: () => void; + readonly #report: (error: unknown) => void; readonly #replace: (() => T) | undefined; - readonly #closedPromise: Promise; + readonly #closed = Promise.withResolvers(); readonly #pending: WatchFrame[] = []; - #resolveClosed!: (end: WatchEnd) => void; #value: T; #listener: ((value: T, ops: readonly Op[], context: Context) => Promise) | undefined; #started = false; #scheduled = false; #running = false; - #detached = false; #retired = false; + /** Set once, by `#terminate()`. */ #end: WatchEnd | undefined; - #resolved = false; #cancellationSignal: AbortSignal | undefined; #cancellationListener: (() => void) | undefined; - /** `replace` gives the value an overflow delivers; by default the newest value. */ - constructor(value: T, detach: () => void, replace?: () => T) { + /** + * `report` gets a listener's throw, which also ends the watch. `replace` gives the value an overflow delivers; by + * default the newest value. + */ + constructor(value: T, detach: () => void, report: (error: unknown) => void, replace?: () => T) { this.#value = value; this.#detach = detach; + this.#report = report; this.#replace = replace; - this.#closedPromise = new Promise((resolve) => { - this.#resolveClosed = resolve; - }); } get value(): T { @@ -185,7 +187,7 @@ export class CommittedWatch implements WatchHandle } get closed(): Promise { - return this.#closedPromise; + return this.#closed.promise; } start(listener: (value: T, ops: readonly Op[], context: Context) => Promise): void { @@ -198,7 +200,7 @@ export class CommittedWatch implements WatchHandle stop(): Promise { this.#terminate({ reason: "stopped" }); - return this.#closedPromise; + return this.#closed.promise; } observeCancellation(signal: AbortSignal): void { @@ -214,8 +216,11 @@ export class CommittedWatch implements WatchHandle this.#terminate({ reason: "cancelled" }); } - closeSession(): void { - this.#terminate({ reason: "session_closed" }); + /** End with `session_closed`, or with `session_failed` and the error when the Session failed. */ + closeSession(failure?: { readonly error: unknown }): void { + this.#terminate( + failure === undefined ? { reason: "session_closed" } : { reason: "session_failed", error: failure.error }, + ); } advance(value: T, ops: readonly Op[], context: Context): void { @@ -241,10 +246,7 @@ export class CommittedWatch implements WatchHandle } async #drain(): Promise { - if (this.#running || this.#end !== undefined || !this.#started) { - this.#finishIfReady(); - return; - } + if (this.#running || this.#end !== undefined || !this.#started) return; this.#running = true; try { while (this.#end === undefined) { @@ -256,6 +258,7 @@ export class CommittedWatch implements WatchHandle await this.#listener!(frame.value, frame.ops, deliveryContext); } catch (error) { if (this.#end === undefined) { + this.#report(error); this.#terminate({ reason: "listener_error", error: toError(error) }); } break; @@ -268,34 +271,22 @@ export class CommittedWatch implements WatchHandle } finally { this.#running = false; if (this.#end === undefined && this.#pending.length > 0) this.#schedule(); - this.#finishIfReady(); } } + /** End the watch, once: detach, drop pending frames, and settle `closed`. A callback already running keeps running. */ #terminate(end: WatchEnd): void { if (this.#end !== undefined) return; this.#end = end; - this.#detachNow(); - this.#pending.length = 0; - this.#finishIfReady(); - } - - #detachNow(): void { - if (this.#detached) return; - this.#detached = true; this.#detach(); - } - - #finishIfReady(): void { - if (this.#resolved || this.#end === undefined) return; - this.#resolved = true; - if (this.#cancellationSignal !== undefined && this.#cancellationListener !== undefined) { - this.#cancellationSignal.removeEventListener("abort", this.#cancellationListener); + this.#pending.length = 0; + if (this.#cancellationListener !== undefined) { + this.#cancellationSignal?.removeEventListener("abort", this.#cancellationListener); } - this.#resolveClosed(this.#end); + this.#closed.resolve(end); } } function toError(error: unknown): Error { - return error instanceof Error ? error : new Error(String(error)); + return error instanceof Error ? error : new Error(errorMessage(error)); } diff --git a/packages/durable/src/session/session.ts b/packages/durable/src/session/session.ts index 571d60858..327514a0d 100644 --- a/packages/durable/src/session/session.ts +++ b/packages/durable/src/session/session.ts @@ -1,5 +1,5 @@ import { type Context, type JsonValue, replicatedState } from "@earendil-works/chord"; -import { awaitWithContext, withoutAbortSignal } from "@earendil-works/chord/context"; +import { awaitWithContext, BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { type Op, track } from "@earendil-works/chord/delta"; import { type AnyDocToken, @@ -8,7 +8,7 @@ import { materializeDocument, resolveAddress, } from "../documents.ts"; -import { StorageRejected } from "../errors.ts"; +import { SessionFailed, StorageRequestError } from "../errors.ts"; import { idFromNumber } from "../ids.ts"; import type { CommitChange, @@ -16,25 +16,41 @@ import type { ConversationDocFamilyToken, ConversationDocToken, ConversationId, + ConversationQuery, ConversationRecord, + Cursor, DocumentAddress, DocumentCommitChange, + DocumentId, + DocumentPoint, + DocumentQuery, DocumentRecord, DocumentState, DocumentWatch, EntryId, + EntryQuery, + EntryRecord, + Id, JsonObject, + Page, RewindableConversationDocFamilyToken, RewindableConversationDocToken, Seq, Session, SessionDocFamilyToken, SessionDocToken, + SessionEnd, Storage, StorageWrite, + StoredDocument, + SubmissionId, + SubmissionQuery, + SubmissionRecord, TaskDocFamilyToken, TaskDocToken, TaskId, + TaskQuery, + TaskRecord, Tx, } from "../types.ts"; import { @@ -45,7 +61,10 @@ import { } from "./observation.ts"; import { type LoadedDocument, Transaction, type TransactionHost, type TransactionScope } from "./transaction.ts"; -/** Open a Session kernel over one storage backend. `now` is the wall clock for task times; default `Date.now`. */ +/** + * Open a Session kernel over one storage backend. `now` is the wall clock for task times; default `Date.now`. The first + * error a Storage method throws fails the Session (`SessionFailed`); see `SessionImpl`. + */ export function createSession(storage: Storage, options?: { readonly now?: () => number }): Session { return new SessionImpl(storage, options?.now); } @@ -55,21 +74,48 @@ export function createSession(storage: Storage, options?: { readonly now?: () => * * Only committed state is observable. Every commit callback, preparation, Storage settlement, adoption, and * publication enqueue runs while the line is held; listeners run later. + * + * Storage failures are final. Every Storage call goes through one guard: the first error a Storage method throws, other + * than for a cancelled caller, fails the Session, as does a commit it cannot adopt. That call gets the error; every later + * call and every Storage call still underway gets `SessionFailed`; close listeners run at once with the error, and the + * Session then closes itself, Storage included, and `closed` settles. Nothing retries: transient errors are the + * Storage's to retry. Reopening recovers from what was committed. */ export class SessionImpl implements Session { readonly #storage: Storage; readonly #documents = new Map(); readonly #commitListeners = new Set<(publication: CommitPublication, context: Context) => void>(); + /** The Session's own listeners, such as its scheduler's: they keep memory in step with storage, so a throw fails it. */ + readonly #internalListeners = new Set<(publication: CommitPublication, context: Context) => void>(); readonly #closeListeners = new Set<() => void>(); readonly #host: TransactionHost; #tail: Promise = Promise.resolve(); #closing: Promise | undefined; - #poison: { readonly error: unknown } | undefined; + #failure: { readonly error: unknown } | undefined; + /** Set when close reaches the backend: calls that fail from then on fail because it closes. */ + readonly #failed = Promise.withResolvers(); + readonly #closed = Promise.withResolvers(); + + get closed(): Promise { + return this.#closed.promise; + } + + /** Internal: the error that failed the Session, if one did. */ + get failure(): { readonly error: unknown } | undefined { + return this.#failure; + } + + /** Internal: rejects with `SessionFailed` the moment the Session fails, before work underway has ended; never resolves. */ + get failed(): Promise { + return this.#failed.promise; + } constructor(storage: Storage, now: () => number = Date.now) { - this.#storage = storage; + // Waits race it; none has to. + this.#failed.promise.catch(() => {}); + this.#storage = new GuardedStorage(storage, this); this.#host = { - storage, + storage: this.#storage, now, cached: (id) => this.#documents.get(id), load: (definition, addressId, address, context) => this.#loadDocument(definition, addressId, address, context), @@ -83,10 +129,30 @@ export class SessionImpl implements Session { }; } + /** The Storage behind the failure guard; every component reads and writes through it. */ + protected get storage(): Storage { + return this.#storage; + } + commit(change: (tx: Tx) => T | Promise, context: Context): Promise { return this.commitWith(change, context); } + /** + * Internal: fail the Session with `error`, the first wins, and report it once. Admission ends, and close listeners run + * now; they read the error from `failure`. The Storage guard calls this; the Harness also calls it for a throw in its + * scheduler's own commits. + */ + fail(error: unknown): void { + if (this.#failure !== undefined) return; + this.#failure = { error }; + this.#failed.reject(new SessionFailed(error)); + // Seal first, so a report handler that calls back finds the Session failed. Closing runs the close listeners + // synchronously; a failing backend close is the caller's to see, and here nobody waits. + this.close(BACKGROUND_CONTEXT).catch(() => {}); + this.report(error); + } + /** * Internal commit exposing the concrete transaction and its internal operations, such as the reserved-ID root * bootstrap and task replacement. `scope` sets the default `tx.createTask()` conversation and the task attributed to @@ -94,7 +160,7 @@ export class SessionImpl implements Session { */ commitWith(change: (tx: Transaction) => T | Promise, context: Context, scope?: TransactionScope): Promise { try { - this.#assertUsable(); + this.assertUsable(); } catch (error) { return Promise.reject(error); } @@ -104,7 +170,7 @@ export class SessionImpl implements Session { /** Internal: run a read-only job on the mutation line so multi-read derivations observe one committed state. */ readOnLine(job: () => Promise): Promise { try { - this.#assertUsable(); + this.assertUsable(); } catch (error) { return Promise.reject(error); } @@ -161,7 +227,7 @@ export class SessionImpl implements Session { context: Context, ): Promise | undefined>; async snapshot(token: AnyDocToken, ...args: readonly unknown[]): Promise { - this.#assertUsable(); + this.assertUsable(); const definition = token.definition; const resolved = resolveAddress(definition, args); const context = args[resolved.nextArgument] as Context; @@ -212,7 +278,7 @@ export class SessionImpl implements Session { ): Promise | undefined>; documentState(token: AnyDocToken, ...args: readonly unknown[]): Promise | undefined> { try { - this.#assertUsable(); + this.assertUsable(); const definition = token.definition; const resolved = resolveAddress(definition, args); const context = args[resolved.nextArgument] as Context; @@ -226,7 +292,7 @@ export class SessionImpl implements Session { (value, release) => new CommittedStateSource(value, release), ); try { - return replicatedState(source) as DocumentState; + return replicatedState(source, { onError: (error) => this.report(error) }) as DocumentState; } catch (error) { detach(); throw error; @@ -266,7 +332,7 @@ export class SessionImpl implements Session { context: Context, ): Promise | undefined>; async watchDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise | undefined> { - this.#assertUsable(); + this.assertUsable(); const definition = token.definition; const resolved = resolveAddress(definition, args); const context = args[resolved.nextArgument] as Context; @@ -286,7 +352,8 @@ export class SessionImpl implements Session { return this.#attachDocument( definition, loaded, - (value, release) => new CommittedWatch(value, release), + (value, release) => + new CommittedWatch(value, release, (error) => this.report(error)), ).observer; }); if (watch === undefined) return undefined; @@ -315,7 +382,7 @@ export class SessionImpl implements Session { context: Context, ): Promise | undefined>; async snapshotAsOf(token: AnyDocToken, ...args: readonly unknown[]): Promise { - this.#assertUsable(); + this.assertUsable(); const definition = token.definition; const resolved = resolveAddress(definition, args); if (resolved.address.scope.kind !== "conversation") { @@ -357,17 +424,41 @@ export class SessionImpl implements Session { .then(() => this.#enqueue(async () => { this.#commitListeners.clear(); + this.#internalListeners.clear(); this.#documents.clear(); - await this.#storage.close(cleanup); + try { + await this.#storage.close(cleanup); + } catch (error) { + // A Storage that cannot close is a failed one; an earlier failure stays the cause. + if (this.#failure === undefined) { + this.#failure = { error }; + this.report(error); + } + throw error; + } }), - ); - const listeners = [...this.#closeListeners]; - this.#closeListeners.clear(); - for (const listener of listeners) listener(); + ) + .finally(() => { + const failure = this.#failure; + this.#closed.resolve( + failure === undefined ? { reason: "closed" } : { reason: "failed", error: failure.error }, + ); + }); + this.#notifyClose(); } return awaitWithContext(this.#closing, context); } + /** Run the close listeners once, each on its own: one that fails is reported, and the rest still run. */ + #notifyClose(): void { + const listeners = [...this.#closeListeners]; + this.#closeListeners.clear(); + for (const listener of listeners) contained(listener, (error) => this.report(error)); + } + + /** Internal: the Session's failure, and errors of listeners, which never fail what ran them. A plain Session drops them. */ + report(_error: unknown): void {} + /** * Runs inside every transaction that creates or forks a conversation, after the conversation record is staged. A * plain Session stages nothing; a Harness stages its built-in documents. @@ -381,16 +472,29 @@ export class SessionImpl implements Session { return Promise.resolve(); } - /** Register a synchronous post-adoption listener. It must not throw, block, or call Session operations. */ + /** Register a synchronous post-adoption listener. It must not block or call Session operations; a throw is reported. */ subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void { - this.#assertUsable(); + this.assertUsable(); this.#commitListeners.add(listener); return () => this.#commitListeners.delete(listener); } - /** Register a listener called synchronously when close begins. It must not throw, block, or call Session operations. */ + /** + * Internal: `subscribeCommits()` for the Session's own components, whose state follows each commit. A throw leaves + * that state behind storage, so it fails the Session instead of being reported. They run before host listeners. + */ + observeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void { + this.assertUsable(); + this.#internalListeners.add(listener); + return () => this.#internalListeners.delete(listener); + } + + /** + * Register a listener called synchronously when close begins, also when a failure closes the Session (`failure` is + * then set). It must not block or call Session operations; a throw is reported. + */ subscribeClose(listener: () => void): () => void { - this.#assertUsable(); + this.assertUsable(); this.#closeListeners.add(listener); return () => this.#closeListeners.delete(listener); } @@ -413,6 +517,8 @@ export class SessionImpl implements Session { let result: T; try { result = await change(tx); + // A callback that caught a failed read must not commit, nor succeed as if it had. + this.#assertHealthy(); } catch (error) { await tx.settleFailure(); throw error; @@ -427,9 +533,8 @@ export class SessionImpl implements Session { // Once admitted, caller cancellation does not interrupt Storage settlement. seq = await this.#storage.commit(writes, withoutAbortSignal(context)); } catch (error) { + // The guard has failed the Session. tx.discard(); - // Callback errors never reach this branch; StorageRejected alone guarantees that no batch effect committed. - if (!(error instanceof StorageRejected)) this.#poison = { error }; throw error; } let documents: DocumentCommitChange[]; @@ -437,7 +542,7 @@ export class SessionImpl implements Session { documents = tx.adopt(seq); } catch (error) { // Storage already committed; a failed adoption leaves memory behind durable state. - this.#poison = { error }; + this.fail(error); throw error; } this.#publish(seq, writes, documents, context); @@ -450,7 +555,7 @@ export class SessionImpl implements Session { documents: readonly DocumentCommitChange[], context: Context, ): void { - if (this.#commitListeners.size === 0) return; + if (this.#commitListeners.size === 0 && this.#internalListeners.size === 0) return; const changes: CommitChange[] = []; for (const write of writes) { switch (write.type) { @@ -463,7 +568,19 @@ export class SessionImpl implements Session { } for (const document of documents) changes.push(document); const publication: CommitPublication = { seq, changes }; - for (const listener of [...this.#commitListeners]) listener(publication, context); + // The commit is durable: a failing listener neither fails the commit nor skips the others. + for (const listener of [...this.#internalListeners]) { + contained( + () => listener(publication, context), + (error) => this.fail(error), + ); + } + for (const listener of [...this.#commitListeners]) { + contained( + () => listener(publication, context), + (error) => this.report(error), + ); + } } /** @@ -485,7 +602,7 @@ export class SessionImpl implements Session { }; const observer = create(loaded.tracker.value, detach); const observed = { version: loaded.valueVersion }; - unsubscribeCommit = this.subscribeCommits((publication, context) => { + unsubscribeCommit = this.observeCommits((publication, context) => { for (const change of publication.changes) { if (change.type !== "document" || change.record.id !== loaded.record.id) continue; // A document state's frames carry no caller cancellation; a watch observes its own cancellation. @@ -496,7 +613,7 @@ export class SessionImpl implements Session { observer.advance(change.value, ops, frameContext); } }); - unsubscribeClose = this.subscribeClose(() => observer.closeSession()); + unsubscribeClose = this.subscribeClose(() => observer.closeSession(this.#failure)); return { observer, detach }; } @@ -536,17 +653,28 @@ export class SessionImpl implements Session { return run; } - #assertUsable(): void { - if (this.#closing !== undefined) throw new Error("Session is closed"); + /** Internal: throw `SessionFailed` when failed, else an error when closing. */ + assertUsable(): void { this.#assertHealthy(); + if (this.#closing !== undefined) throw new Error("Session is closed"); } #assertHealthy(): void { - if (this.#poison !== undefined) { - throw new Error("Session is poisoned by a failed commit after storage admission; reopen it", { - cause: this.#poison.error, - }); - } + if (this.#failure !== undefined) throw new SessionFailed(this.#failure.error); + } +} + +/** + * Run a host callback that must not fail its caller: a throw, or a promise it returns that rejects, goes to `onError`, + * which is not awaited. + */ +export function contained(callback: () => unknown, onError: (error: unknown) => void): void { + try { + const result = callback(); + // Any thenable, also a promise from another realm; `Promise.resolve` adopts it with handlers of its own. + if (typeof (result as { then?: unknown } | null)?.then === "function") Promise.resolve(result).catch(onError); + } catch (error) { + onError(error); } } @@ -567,3 +695,180 @@ function observedOperations( function cancellationError(signal: AbortSignal): Error { return signal.reason instanceof Error ? signal.reason : new DOMException("The operation was aborted", "AbortError"); } + +/** What the guard needs of its Session: the failure so far, and how to fail it. */ +type FailureLatch = { + readonly failure: { readonly error: unknown } | undefined; + fail(error: unknown): void; +}; + +/** + * The Session's Storage behind its failure guard; every component of the Session reads and writes through it. Before a + * call, a failed Session gets `SessionFailed` without reaching the backend. A call that throws fails the Session and + * rethrows the backend's error, unless it is a `StorageRequestError` or a read whose caller's context was aborted: + * neither says the Storage is broken. + * `close()` always reaches the backend, so a failed Session still closes it. + */ +class GuardedStorage implements Storage { + readonly #storage: Storage; + readonly #latch: FailureLatch; + /** Backend calls underway, also those off the Session line, which close waits for before it closes the backend. */ + readonly #underway = new Set>(); + #closing = false; + + constructor(storage: Storage, latch: FailureLatch) { + this.#storage = storage; + this.#latch = latch; + } + + /** Never exempt, whatever it throws: once admitted, whether a failed batch committed is unknown. */ + commit(writes: readonly StorageWrite[], context: Context): Promise { + return this.#call(undefined, () => this.#storage.commit(writes, context)); + } + + mintId>(): Promise { + return this.#call(undefined, () => this.#storage.mintId()); + } + + conversation(id: ConversationId, context: Context): Promise { + return this.#call(context, () => this.#storage.conversation(id, context)); + } + + scanConversations( + query: ConversationQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#call(context, () => this.#storage.scanConversations(query, limit, cursor, context)); + } + + entry(id: EntryId, context: Context): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>; + entry( + conversationId: ConversationId, + id: EntryId, + context: Context, + ): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined>; + entry( + first: EntryId | ConversationId, + second: EntryId | Context, + third?: Context, + ): Promise<{ readonly entry: EntryRecord; readonly commitSeq: Seq } | undefined> { + if (third === undefined) { + const context = second as Context; + return this.#call(context, () => this.#storage.entry(first as EntryId, context)); + } + return this.#call(third, () => this.#storage.entry(first as ConversationId, second as EntryId, third)); + } + + findLatestHeadMarker( + conversationId: ConversationId, + atOrBeforeEntryId: EntryId | undefined, + context: Context, + ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> { + return this.#call(context, () => this.#storage.findLatestHeadMarker(conversationId, atOrBeforeEntryId, context)); + } + + scanEntries( + query: EntryQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#call(context, () => this.#storage.scanEntries(query, limit, cursor, context)); + } + + task(id: TaskId, context: Context): Promise | undefined> { + return this.#call(context, () => this.#storage.task(id, context)); + } + + scanTasks( + query: TaskQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise, Cursor>> { + return this.#call(context, () => this.#storage.scanTasks(query, limit, cursor, context)); + } + + submission(id: SubmissionId, context: Context): Promise { + return this.#call(context, () => this.#storage.submission(id, context)); + } + + scanSubmissions( + query: SubmissionQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#call(context, () => this.#storage.scanSubmissions(query, limit, cursor, context)); + } + + submissionByRequest( + conversationId: ConversationId, + requestId: string, + context: Context, + ): Promise { + return this.#call(context, () => this.#storage.submissionByRequest(conversationId, requestId, context)); + } + + findDocument(address: DocumentAddress, at: DocumentPoint, context: Context): Promise { + return this.#call(context, () => this.#storage.findDocument(address, at, context)); + } + + document(id: DocumentId, at: DocumentPoint, context: Context): Promise { + return this.#call(context, () => this.#storage.document(id, at, context)); + } + + scanDocuments( + query: DocumentQuery, + limit: number, + cursor: Cursor | undefined, + context: Context, + ): Promise> { + return this.#call(context, () => this.#storage.scanDocuments(query, limit, cursor, context)); + } + + /** + * Close the backend once every call underway has settled, so none outlives it or fails after `closed` settles. New + * calls are refused from here on: a paged read must not start its next page while the backend closes. + */ + async close(context: Context): Promise { + this.#closing = true; + await Promise.allSettled([...this.#underway]); + return this.#storage.close(context); + } + + /** + * Run one backend call under the guard. `read` is a read's context; `commit()` and `mintId()` pass none, so nothing + * exempts their errors: once a batch is admitted, whether it committed is unknown. A call still underway when another + * fails the Session ends with `SessionFailed` too, whatever it returns: nothing it read or wrote is used. + */ + async #call(read: Context | undefined, run: () => Promise): Promise { + this.#assertHealthy(); + if (this.#closing) throw new Error("Session is closed"); + let result: T; + // An async wrapper, so a backend that throws synchronously is guarded too. + const call = (async () => run())(); + this.#underway.add(call); + try { + result = await call; + } catch (error) { + this.#assertHealthy(); + // An invalid read, or one its caller cancelled, says nothing about the Storage; it fails that call only. + const exempt = + read !== undefined && (error instanceof StorageRequestError || read.abortSignal?.aborted === true); + if (!exempt) this.#latch.fail(error); + throw error; + } finally { + this.#underway.delete(call); + } + this.#assertHealthy(); + return result; + } + + #assertHealthy(): void { + const failure = this.#latch.failure; + if (failure !== undefined) throw new SessionFailed(failure.error); + } +} diff --git a/packages/durable/src/session/transaction.ts b/packages/durable/src/session/transaction.ts index 4c0bfd3e6..d97d56418 100644 --- a/packages/durable/src/session/transaction.ts +++ b/packages/durable/src/session/transaction.ts @@ -416,6 +416,7 @@ export class Transaction implements Tx { ...(owner === undefined ? {} : { owner: owner.id }), background: options.background ?? false, abortRequested: false, + ...(options.abandonOnRestart === true ? { abandonOnRestart: true } : {}), state: { status: "pending", checkpoint }, }, TABLE_JSON_COPY_OPTIONS, diff --git a/packages/durable/src/storage/jsonl/storage.ts b/packages/durable/src/storage/jsonl/storage.ts index a059cff8f..4b03fd5be 100644 --- a/packages/durable/src/storage/jsonl/storage.ts +++ b/packages/durable/src/storage/jsonl/storage.ts @@ -1,5 +1,6 @@ import type { Context, JsonValue } from "@earendil-works/chord"; import type { FileError, FileSystem } from "../../env/index.ts"; +import { StorageRequestError } from "../../errors.ts"; import { idFromNumber, seqFromNumber } from "../../ids.ts"; import type { ConversationId, @@ -322,7 +323,7 @@ export class JsonlStorage implements Storage { async entry(idOrConversationId: EntryId | ConversationId, idOrContext: EntryId | Context, context?: Context) { if (context === undefined) return this.store.entry(idFromNumber(idOrConversationId), idOrContext as Context); - if (typeof idOrContext !== "number") throw new TypeError("Storage.entry() requires an entry ID"); + if (typeof idOrContext !== "number") throw new StorageRequestError("Storage.entry() requires an entry ID"); return this.store.entry( idFromNumber(idOrConversationId), idFromNumber(idOrContext), diff --git a/packages/durable/src/storage/memory.ts b/packages/durable/src/storage/memory.ts index 7333d37f9..d6e3b80c4 100644 --- a/packages/durable/src/storage/memory.ts +++ b/packages/durable/src/storage/memory.ts @@ -1,6 +1,6 @@ import type { Context, JsonValue } from "@earendil-works/chord"; import { applyImmutableBatches, type Op } from "@earendil-works/chord/delta"; -import { StorageRejected } from "../errors.ts"; +import { StorageRequestError } from "../errors.ts"; import { idFromNumber, seqFromNumber } from "../ids.ts"; import type { ConversationId, @@ -122,7 +122,8 @@ const freeze = (value: T): T => { const cursorId = >(cursor: Readonly> | undefined): I | undefined => { const after = cursor?.after; if (after === undefined) return undefined; - if (typeof after !== "number" || !Number.isSafeInteger(after)) throw new TypeError("Invalid storage cursor"); + if (typeof after !== "number" || !Number.isSafeInteger(after)) + throw new StorageRequestError("Invalid storage cursor"); return idFromNumber(after); }; @@ -321,8 +322,7 @@ export class MemoryStorage implements Storage { content: { kind: "base", version: stored.version, value: stored.value }, }; } catch (error) { - if (error instanceof StorageRejected) throw error; - throw new StorageRejected(`Document copy ${write.record.id} was rejected`, { cause: error }); + throw new Error(`Document copy ${write.record.id} was rejected`, { cause: error }); } }); } @@ -480,7 +480,7 @@ export class MemoryStorage implements Storage { if (entry === undefined) return undefined; return { entry: clone(entry), commitSeq: this.state.entryCommitSeqs.get(id)! }; } - if (typeof idOrContext !== "number") throw new TypeError("Storage.entry() requires an entry ID"); + if (typeof idOrContext !== "number") throw new StorageRequestError("Storage.entry() requires an entry ID"); const conversationId = idFromNumber(idOrConversationId); const id = idFromNumber(idOrContext); const entry = this.visibleEntries(conversationId, id, id).next().value; @@ -495,7 +495,7 @@ export class MemoryStorage implements Storage { ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> { this.assertOpen(); if (!this.state.conversations.has(conversationId)) { - throw new Error(`Unknown conversation: ${conversationId}`); + throw new StorageRequestError(`Unknown conversation: ${conversationId}`); } let currentId = conversationId; let upperEntryId = atOrBeforeEntryId ?? Number.POSITIVE_INFINITY; @@ -660,7 +660,7 @@ export class MemoryStorage implements Storage { const stored = this.state.documents.get(id); if (stored === undefined) return undefined; if (at !== "current" && isCurrentOnly(stored.record)) { - throw new Error(`Document ${id} does not retain historical content`); + throw new StorageRequestError(`Document ${id} does not retain historical content`); } if (!isAliveAt(stored.record, at)) return undefined; const revisions = at === "current" ? stored.revisions : stored.revisions.filter((revision) => revision.seq <= at); @@ -681,7 +681,7 @@ export class MemoryStorage implements Storage { maxEntryId: number = Number.POSITIVE_INFINITY, ): Generator { if (!this.state.conversations.has(conversationId)) { - throw new Error(`Unknown conversation: ${conversationId}`); + throw new StorageRequestError(`Unknown conversation: ${conversationId}`); } let currentId = conversationId; let upperEntryId = maxEntryId; @@ -707,7 +707,7 @@ export class MemoryStorage implements Storage { maxEntryId: number = Number.POSITIVE_INFINITY, ): Generator { if (!this.state.conversations.has(conversationId)) { - throw new Error(`Unknown conversation: ${conversationId}`); + throw new StorageRequestError(`Unknown conversation: ${conversationId}`); } const segments: { readonly conversationId: ConversationId; readonly upper: number }[] = []; let currentId = conversationId; diff --git a/packages/durable/src/storage/scan.ts b/packages/durable/src/storage/scan.ts index 8cbc1938c..ed6ec8951 100644 --- a/packages/durable/src/storage/scan.ts +++ b/packages/durable/src/storage/scan.ts @@ -1,3 +1,4 @@ +import { StorageRequestError } from "../errors.ts"; import type { Id, ScanOrder } from "../types.ts"; /** Where a built-in storage scan starts: its order and the last ID a previous page returned. */ @@ -17,17 +18,18 @@ export function scanStart( fallback: ScanOrder, ): ScanStart { if (requested !== undefined && requested !== "ascending" && requested !== "descending") { - throw new TypeError(`Invalid scan order: ${String(requested)}`); + throw new StorageRequestError(`Invalid scan order: ${String(requested)}`); } if (cursor === undefined) return { order: requested ?? fallback, after: undefined }; const { after, order: stored } = cursor; - if (typeof after !== "number" || !Number.isSafeInteger(after)) throw new TypeError("Invalid storage cursor"); + if (typeof after !== "number" || !Number.isSafeInteger(after)) + throw new StorageRequestError("Invalid storage cursor"); if (stored !== undefined && stored !== "ascending" && stored !== "descending") { - throw new TypeError("Invalid storage cursor"); + throw new StorageRequestError("Invalid storage cursor"); } const order = stored ?? fallback; if (requested !== undefined && requested !== order) { - throw new TypeError(`The cursor continues a ${order} scan; the query asks for ${requested}`); + throw new StorageRequestError(`The cursor continues a ${order} scan; the query asks for ${requested}`); } return { order, after }; } diff --git a/packages/durable/src/storage/sqlite/storage.ts b/packages/durable/src/storage/sqlite/storage.ts index 90666efc2..c524246ef 100644 --- a/packages/durable/src/storage/sqlite/storage.ts +++ b/packages/durable/src/storage/sqlite/storage.ts @@ -1,6 +1,6 @@ import type { Context, JsonValue } from "@earendil-works/chord"; import { apply, type Op } from "@earendil-works/chord/delta"; -import { StorageRejected } from "../../errors.ts"; +import { StorageRequestError } from "../../errors.ts"; import { idFromNumber, seqFromNumber } from "../../ids.ts"; import type { ConversationId, @@ -68,7 +68,8 @@ const encodeIndexedString = (value: string): string => JSON.stringify(value); const cursorId = >(cursor: Cursor | undefined): I | undefined => { const after = cursor?.after; if (after === undefined) return undefined; - if (typeof after !== "number" || !Number.isSafeInteger(after)) throw new TypeError("Invalid storage cursor"); + if (typeof after !== "number" || !Number.isSafeInteger(after)) + throw new StorageRequestError("Invalid storage cursor"); return idFromNumber(after); }; @@ -281,12 +282,12 @@ export class SqliteStorage implements Storage { : typeof idOrContext === "number" ? idFromNumber(idOrContext) : undefined; - if (id === undefined) throw new TypeError("Storage.entry() requires an entry ID"); + if (id === undefined) throw new StorageRequestError("Storage.entry() requires an entry ID"); let conversation: ConversationRecord | undefined; if (context !== undefined) { const conversationId = idFromNumber(idOrConversationId); conversation = await this.readConversation(conversationId); - if (conversation === undefined) throw new Error(`Unknown conversation: ${conversationId}`); + if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${conversationId}`); } const row = await this.db.get("SELECT record, commit_seq FROM entries WHERE id = ?", id); if (row === undefined) return undefined; @@ -308,7 +309,7 @@ export class SqliteStorage implements Storage { atOrBeforeEntryId: EntryId | undefined, ): Promise<(EntryRecord & { readonly head: EntryId }) | undefined> { let conversation = await this.readConversation(conversationId); - if (conversation === undefined) throw new Error(`Unknown conversation: ${conversationId}`); + if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${conversationId}`); let upper: number | undefined = atOrBeforeEntryId; while (true) { const row = @@ -337,7 +338,7 @@ export class SqliteStorage implements Storage { const { order, after } = scanStart(query.order, cursor, "descending"); if (order === "ascending") return this.readEntriesAscending(query, limit, after); let conversation = await this.readConversation(query.conversationId); - if (conversation === undefined) throw new Error(`Unknown conversation: ${query.conversationId}`); + if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${query.conversationId}`); let upper: number | undefined = query.maxEntryId; if (after !== undefined) upper = Math.min(upper ?? Number.MAX_SAFE_INTEGER, after - 1); const values: EntryRecord[] = []; @@ -374,7 +375,7 @@ export class SqliteStorage implements Storage { ): Promise> { const segments: { readonly conversationId: ConversationId; readonly upper: number | undefined }[] = []; let conversation = await this.readConversation(query.conversationId); - if (conversation === undefined) throw new Error(`Unknown conversation: ${query.conversationId}`); + if (conversation === undefined) throw new StorageRequestError(`Unknown conversation: ${query.conversationId}`); let upper: number | undefined = query.maxEntryId; while (true) { segments.push({ conversationId: conversation.id, upper }); @@ -612,7 +613,7 @@ export class SqliteStorage implements Storage { if (row === undefined) return undefined; const record = parseJson(row.record); if (at !== "current" && isCurrentOnly(record)) { - throw new Error(`Document ${id} does not retain historical content`); + throw new StorageRequestError(`Document ${id} does not retain historical content`); } if (!isAliveAt(record, at)) return undefined; const upper = at === "current" ? Number.MAX_SAFE_INTEGER : at; @@ -725,7 +726,7 @@ export class SqliteStorage implements Storage { const liveCounts = new Map(); for (const [id, action] of actions) { if (action.copy !== undefined && actions.has(action.copy.id)) { - throw new StorageRejected(`Document copy ${id} source is changed in the copy batch`); + throw new Error(`Document copy ${id} source is changed in the copy batch`); } const row = await executor.get("SELECT record FROM documents WHERE id = ?", id); const existing = row === undefined ? undefined : parseJson(row.record); @@ -864,8 +865,7 @@ export class SqliteStorage implements Storage { } content = { kind: "base", version: stored.version, value: stored.value }; } catch (error) { - if (error instanceof StorageRejected) throw error; - throw new StorageRejected(`Document copy ${id} was rejected`, { cause: error }); + throw new Error(`Document copy ${id} was rejected`, { cause: error }); } } let record: DocumentRecord; diff --git a/packages/durable/src/testing/storage-conformance.ts b/packages/durable/src/testing/storage-conformance.ts index 753bf2395..ad0f58aad 100644 --- a/packages/durable/src/testing/storage-conformance.ts +++ b/packages/durable/src/testing/storage-conformance.ts @@ -1179,7 +1179,8 @@ export function createStorageConformance(options: StorageConformanceOptions): re } catch (error) { conflictError = error; } - expect((conflictError as Error | undefined)?.name).toBe("StorageRejected"); + // Rejected without effect: the copy and the retirement in the same batch are both absent. + expect(conflictError instanceof Error).toBe(true); expect(await storage.document(conflictId, "current", context)).toBeUndefined(); expect((await storage.document(currentCopyId, "current", context))?.value).toEqual({ count: 2, @@ -1202,7 +1203,7 @@ export function createStorageConformance(options: StorageConformanceOptions): re } catch (error) { mismatchError = error; } - expect((mismatchError as Error | undefined)?.name).toBe("StorageRejected"); + expect(mismatchError instanceof Error).toBe(true); expect(await storage.document(mismatchId, "current", context)).toBeUndefined(); }, ), diff --git a/packages/durable/src/tools/bash.ts b/packages/durable/src/tools/bash.ts index ae2603efe..21d1e1d03 100644 --- a/packages/durable/src/tools/bash.ts +++ b/packages/durable/src/tools/bash.ts @@ -1,7 +1,7 @@ import type { Context } from "@earendil-works/chord"; import { type Static, Type } from "typebox"; import { defineTool } from "../harness/define.ts"; -import type { ToolExecutionApi, ToolRegistration } from "../harness/types.ts"; +import type { ToolExecutionApi, ToolExecutionResult, ToolRegistration } from "../harness/types.ts"; import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "../truncate.ts"; import { requireEnv } from "./env.ts"; @@ -17,8 +17,17 @@ const powershellSchema = Type.Object({ timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })), }); +/** What a program that calls `bash` or `powershell` receives. */ +const shellStructuredOutputSchema = Type.Object({ + output: Type.String({ description: "Combined stdout and stderr: the retained tail, as the model sees it" }), + truncated: Type.Boolean({ description: "Whether earlier output was dropped" }), + fullOutputPath: Type.Optional(Type.String({ description: "File with the complete output, when it was spilled" })), + exitCode: Type.Number(), +}); + export type BashToolInput = Static; export type PowerShellToolInput = Static; +export type ShellStructuredOutput = Static; /** A command about to run, which `prepare` may change: the script, its working directory and environment. */ export interface BashExecution { @@ -72,7 +81,8 @@ async function prepareExecution( /** * Run each command form in turn until one starts, streaming output to `api.output()` within the retained window, and - * turn the result into the tool's outcome: a spill diagnostic, and a thrown error for a failure or a nonzero exit. + * turn the result into the tool's result: a spill diagnostic, an error result for a nonzero exit, and a thrown error for + * a failure such as a timeout. The structured output repeats the retained output with the exit code. */ async function runCommand( commands: readonly (string | readonly string[])[], @@ -80,7 +90,7 @@ async function runCommand( timeout: number | undefined, api: ToolExecutionApi, context: Context, -): Promise { +): Promise { const env = requireEnv(api); let result: Awaited> | undefined; for (const command of commands) { @@ -112,27 +122,37 @@ async function runCommand( if (result.error.code === "aborted") throw new Error("Command aborted"); throw result.error; } - if (result.value.exitCode !== 0) throw new Error(`Command exited with code ${result.value.exitCode}`); + const exitCode = result.value.exitCode; + const retained = api.retainedOutput(); + const structuredOutput: ShellStructuredOutput = { + output: retained.text, + truncated: retained.truncated, + ...(spillPath === undefined ? {} : { fullOutputPath: spillPath }), + exitCode, + }; + if (exitCode === 0) return { structuredOutput }; + const exited = { severity: "error", code: "exit_code", message: `Command exited with code ${exitCode}` } as const; + return { structuredOutput, isError: true, diagnostics: [exited] }; } /** * Runs a command through the environment's shell. Its output streams to `api.output()`, where the Harness keeps the * tail within the default limits; the result content is that retained output. The retained window goes to the * environment, which may omit output outside it and report how much it omitted, so dropped counts stay exact. Output - * beyond the limits is spilled to a file whose path is reported as a diagnostic. A nonzero exit or timeout throws, which - * makes an error result that still carries the output and diagnostics. + * beyond the limits is spilled to a file whose path is reported as a diagnostic. A nonzero exit is an error result with + * the exit code; a timeout throws, which makes an error result that still carries the output and diagnostics. */ export function createBashTool(options?: BashToolOptions): ToolRegistration { return defineTool({ name: "bash", description: `Execute a bash command in the current working directory. Returns combined stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`, parameters: bashSchema, + structuredOutputSchema: shellStructuredOutputSchema, outputLimits: { retain: "tail" }, async execute(args, api, context) { validateTimeout(args.timeout); const execution = await prepareExecution(args.command, options, api, context); - await runCommand([execution.command], execution, args.timeout, api, context); - return {}; + return runCommand([execution.command], execution, args.timeout, api, context); }, }); } @@ -151,14 +171,14 @@ export function createPowerShellTool(options?: PowerShellToolOptions): ToolRegis name: "powershell", description: `Execute a PowerShell command in the current working directory. Returns combined stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`, parameters: powershellSchema, + structuredOutputSchema: shellStructuredOutputSchema, outputLimits: { retain: "tail" }, async execute(args, api, context) { validateTimeout(args.timeout); const execution = await prepareExecution(args.command, options, api, context); const script = `${UTF8_OUTPUT}\n${execution.command}`; const commands = programs.map((program) => [program, ...POWERSHELL_ARGS, script]); - await runCommand(commands, execution, args.timeout, api, context); - return {}; + return runCommand(commands, execution, args.timeout, api, context); }, }); } diff --git a/packages/durable/src/tools/edit.ts b/packages/durable/src/tools/edit.ts index 807b6d568..802150152 100644 --- a/packages/durable/src/tools/edit.ts +++ b/packages/durable/src/tools/edit.ts @@ -133,7 +133,7 @@ export function createEditTool(): ToolRegistration { if (context.abortSignal?.aborted) throw new Error("Operation aborted"); getOrThrow(await env.writeFile(absolutePath, content, context)); if (context.abortSignal?.aborted) throw new Error("Operation aborted"); - return { content: [{ type: "text", text: `Successfully wrote to ${path}` }] }; + return { output: [{ type: "text", text: `Successfully wrote to ${path}` }] }; }, context, ); diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index b2924e82d..d7aa8d513 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -208,6 +208,11 @@ export interface TaskRuntime extends DocumentObserver getTask(id: TaskId, context: Context): Promise | undefined>; /** Resolve with the task's terminal receipt; rejects when the invocation ends. */ waitForTask(id: TaskId, context: Context): Promise>; + /** + * Abort a task this task owns, as `Harness.abortTask()` does, and resolve once it is terminal. Rejects for a task + * another owns; resolves at once for a terminal one. + */ + abortOwned(id: TaskId, context: Context): Promise; /** Outcomes of terminal tasks, in order; rejects when one is missing or not terminal. Used after a wait. */ outcomes(ids: readonly TaskId[], context: Context): Promise[]>; /** @@ -277,6 +282,12 @@ export type TaskOptions = { readonly conversationId?: ConversationId; /** Conversation-owned tasks only: excluded from ordinary idle waits, conversation aborts, and cascades. */ readonly background?: boolean; + /** + * Its creator awaits it only in memory and does not resume after a restart, so it is pointless once the Harness that + * created it is gone. When scheduling starts in a later Harness, the task, if still live, is abort-marked with reason + * `restart`, and the mark cascades to its ordinary owned work, before any of it runs again. + */ + readonly abandonOnRestart?: boolean; }; /** Ownership selected explicitly whenever a conversation is created. */ @@ -535,6 +546,14 @@ type TaskRecordBase = { readonly background: boolean; /** Durable abort mark checked before run-mode progress is committed. */ readonly abortRequested: boolean; + /** + * Why the mark was set when not by an abort request: `restart` for a task abandoned after a restart + * (`TaskOptions.abandonOnRestart`) or below one. Such a task waits for its definition instead of becoming `orphaned`. + * A later abort request clears it. + */ + readonly abortReason?: "restart"; + /** Set from `TaskOptions.abandonOnRestart`. Immutable. */ + readonly abandonOnRestart?: true; /** * Wall-clock milliseconds of the first change to `running`, stamped by the Session. Kept through waits and recovery, * so the span to `endedAt` includes them. Absent before the task first runs, and on records written by earlier @@ -866,7 +885,9 @@ export type DocumentState = AttachedReplicatedState { @@ -917,15 +938,26 @@ export interface DocumentObserver { ): Promise | undefined>; } +/** Why a Session ended: `close()`, or a failed Storage call, whose error it carries. */ +export type SessionEnd = { readonly reason: "closed" } | { readonly reason: "failed"; readonly error: unknown }; + /** Owner of one mutation line, its records, and its tracked documents. */ export interface Session extends DocumentObserver { /** Run one atomic transaction on the Session mutation line. */ commit(change: (tx: Tx) => T | Promise, context: Context): Promise; /** Seal admission, settle admitted commits, then close storage. */ close(context: Context): Promise; - /** Observe complete commits synchronously after adoption. The listener must not throw, block, or call Session APIs. */ + /** + * Settles once the Session has closed, Storage included: after `close()`, or after the first failed Storage call, + * which closes the Session itself. A failed Session is reopened from Storage; nothing else is left to clean up. + */ + readonly closed: Promise; + /** Observe complete commits synchronously after adoption. The listener must not block or call Session APIs. */ subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void; - /** Observe close synchronously when it begins. The listener must not throw, block, or call Session APIs. */ + /** + * Observe close synchronously when it begins, also when a failed Storage call closes the Session. The listener must + * not block or call Session APIs. + */ subscribeClose(listener: () => void): () => void; snapshot(token: SessionDocToken, context: Context): Promise | undefined>; @@ -1011,6 +1043,11 @@ export interface Session extends DocumentObserver { * ancestry, and transitions. Implementations enforce atomicity, global ID ownership, * immutable conversation/entry creation, document record consistency, and detached values; * Session serializes commits. + * + * An error a method throws is final: it fails the Session, which nothing retries, so retry transient failures inside + * the method. Two errors of a read fail only that read: a `StorageRequestError` for an invalid request (an unknown + * conversation, a foreign or malformed cursor, history a document does not keep), which must have no durable effect, and a + * rejection because its caller's context was aborted. Any error from `commit()` or `mintId()` is fatal. */ export interface Storage { /** @@ -1018,7 +1055,10 @@ export interface Storage { */ commit(writes: readonly StorageWrite[], context: Context): Promise; - /** Return a fresh branded candidate from the Session-global numeric ID namespace. */ + /** + * Return a fresh branded candidate from the Session-global numeric ID namespace, greater than every ID minted or stored + * before, also across reopen: scans order by ID as creation order, and the Harness relies on it. + */ mintId>(): Promise; /** Look up one conversation by exact ID. */ diff --git a/packages/durable/test/examples/06-harness.ts b/packages/durable/test/examples/06-harness.ts index ef672f153..141e8e667 100644 --- a/packages/durable/test/examples/06-harness.ts +++ b/packages/durable/test/examples/06-harness.ts @@ -24,7 +24,7 @@ const read = defineTool({ name: "read", description: "Read a file", parameters: Type.Object({ path: Type.String() }), - execute: async (args) => ({ content: [{ type: "text", text: `contents of ${args.path}` }] }), + execute: async (args) => ({ output: [{ type: "text", text: `contents of ${args.path}` }] }), }); const Files = defineExtension({ name: "files", tools: [read] }); const registry = createRegistry(); diff --git a/packages/durable/test/examples/07-configuration.ts b/packages/durable/test/examples/07-configuration.ts index 0e760ed93..1d6a7c019 100644 --- a/packages/durable/test/examples/07-configuration.ts +++ b/packages/durable/test/examples/07-configuration.ts @@ -26,7 +26,7 @@ function exampleTool(name: string, description: string): AppTool { name, description, parameters: Type.Object({ path: Type.String() }), - execute: async (args) => ({ content: [{ type: "text", text: `${name} ${args.path}` }] }), + execute: async (args) => ({ output: [{ type: "text", text: `${name} ${args.path}` }] }), }); return { ...tool, snippet: `Use ${name} for files.` }; } diff --git a/packages/durable/test/examples/10-registry-reload.ts b/packages/durable/test/examples/10-registry-reload.ts index bf3bd4c30..97c668be0 100644 --- a/packages/durable/test/examples/10-registry-reload.ts +++ b/packages/durable/test/examples/10-registry-reload.ts @@ -13,7 +13,7 @@ function exampleTool(name: string, description: string) { name, description, parameters: Type.Object({ path: Type.String() }), - execute: async (args) => ({ content: [{ type: "text", text: `${name} ${args.path}` }] }), + execute: async (args) => ({ output: [{ type: "text", text: `${name} ${args.path}` }] }), }); } diff --git a/packages/durable/test/examples/11-extension-state.ts b/packages/durable/test/examples/11-extension-state.ts index 3f41a3cac..a75a7ef4a 100644 --- a/packages/durable/test/examples/11-extension-state.ts +++ b/packages/durable/test/examples/11-extension-state.ts @@ -42,7 +42,7 @@ const Todo = defineExtension({ await api.commit(async (tx) => { (await tx.doc(Todos, api.conversationId)).items.push(args.item); }, callContext); - return { content: [{ type: "text", text: `added ${args.item}` }] }; + return { output: [{ type: "text", text: `added ${args.item}` }] }; }, }), ], diff --git a/packages/durable/test/examples/22-subagent-foreground.ts b/packages/durable/test/examples/22-subagent-foreground.ts index 060ded2db..69d7ec5c4 100644 --- a/packages/durable/test/examples/22-subagent-foreground.ts +++ b/packages/durable/test/examples/22-subagent-foreground.ts @@ -70,7 +70,7 @@ const Subagent: Extension = defineExtension({ throw new Error(`Subagent failed: ${settled.status}`); } const text = await answerText(api, settled.answer, callContext); - return { content: [{ type: "text", text }], details: { conversationId: child } }; + return { output: [{ type: "text", text }], details: { conversationId: child } }; }, }), ], diff --git a/packages/durable/test/examples/27-plan-mode.ts b/packages/durable/test/examples/27-plan-mode.ts index 05c621cd1..ebd6438da 100644 --- a/packages/durable/test/examples/27-plan-mode.ts +++ b/packages/durable/test/examples/27-plan-mode.ts @@ -43,7 +43,7 @@ const submitPlan = defineTool({ await api.commit(async (tx) => { (await tx.doc(PlanDoc, api.conversationId)).steps = args.steps; }, callContext); - return { content: [{ type: "text", text: "Plan submitted." }], control: { terminate: true } }; + return { output: [{ type: "text", text: "Plan submitted." }], control: { terminate: true } }; }, }); diff --git a/packages/durable/test/examples/31-reload-and-restart.ts b/packages/durable/test/examples/31-reload-and-restart.ts index 727c6039f..3da03b0b2 100644 --- a/packages/durable/test/examples/31-reload-and-restart.ts +++ b/packages/durable/test/examples/31-reload-and-restart.ts @@ -28,7 +28,7 @@ function loadVersioned(version: string) { execute: async () => { started(); await gate; - return { content: [{ type: "text", text: version }] }; + return { output: [{ type: "text", text: version }] }; }, }), ], diff --git a/packages/durable/test/harness-callers.test.ts b/packages/durable/test/harness-callers.test.ts new file mode 100644 index 000000000..ff9a8311e --- /dev/null +++ b/packages/durable/test/harness-callers.test.ts @@ -0,0 +1,144 @@ +import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; +import { + AgentDoc, + type Conversation, + defineTool, + MemoryStorage, + type NestedToolExecutionResult, + type ToolRegistration, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; +import { addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; + +function tool(name: string, extra: Partial = {}): ToolRegistration { + return defineTool({ + name, + description: name, + parameters: Type.Object({}), + execute: async () => ({ output: [{ type: "text", text: `${name} ran` }] }), + ...extra, + }); +} + +const DONE = fauxAssistantMessage([fauxText("done")]); + +function call(name: string, id = "c1"): AssistantMessage { + return fauxAssistantMessage([fauxToolCall(name, {}, { id })], { stopReason: "toolUse" }); +} + +/** A tool that calls each of `names` as a nested call and records what it got back. */ +function prober(names: readonly string[]) { + const received = new Map(); + const registration = defineTool({ + name: "probe", + description: "probe", + parameters: Type.Object({}), + execute: async (_args, api, callContext) => { + for (const name of names) received.set(name, await api.executeTool(name, {}, callContext)); + return {}; + }, + }); + return { registration, received }; +} + +async function offered(root: Conversation): Promise { + return (await root.agent(context)).tools.map((each) => each.name); +} + +async function callable(root: Conversation): Promise { + return (await root.agent(context)).callable.map((each) => each.name); +} + +async function setupWith(tools: readonly ToolRegistration[]): Promise { + const setup = chatSetup(); + for (const each of tools) addTool(setup.registry, each); + return setup; +} + +describe("who may call a tool", () => { + it("offers the model only tools it may call, and lets tools call only tools they may call", async () => { + const script = tool("script", { callers: ["model"] }); + const hidden = tool("hidden", { callers: ["tools"] }); + const both = tool("both"); + const probe = prober(["script", "hidden", "both"]); + const setup = await setupWith([script, hidden, both, probe.registration]); + setup.faux.setResponses([call("probe"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + expect(await offered(root)).toEqual(["script", "both", "probe"]); + expect(await callable(root)).toEqual(["hidden", "both", "probe"]); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(probe.received.get("script")?.diagnostics[0]?.code).toBe("tool_unavailable"); + expect(probe.received.get("hidden")).toMatchObject({ isError: false, structuredOutput: "hidden ran" }); + expect(probe.received.get("both")).toMatchObject({ isError: false, structuredOutput: "both ran" }); + await harness.close(context); + }); + + it("narrows what the model is offered with modelTools, leaving the rest callable by tools", async () => { + const read = tool("read"); + const bash = tool("bash"); + const probe = prober(["read", "bash"]); + const setup = await setupWith([read, bash, probe.registration]); + setup.faux.setResponses([call("probe"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ modelTools: [probe.registration] }, context); + expect(await offered(root)).toEqual(["probe"]); + expect(await callable(root)).toEqual(["read", "bash", "probe"]); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(probe.received.get("read")?.isError).toBe(false); + expect(probe.received.get("bash")?.isError).toBe(false); + + await root.configure({ modelTools: { remove: [bash] } }, context); + expect(await offered(root)).toEqual(["read", "probe"]); + expect((await harness.snapshot(AgentDoc, root.id, context))?.modelTools).toEqual({ remove: ["bash"] }); + await root.configure({ modelTools: null }, context); + expect(await offered(root)).toEqual(["read", "bash", "probe"]); + await harness.close(context); + }); + + it("never offers a tool that is not enabled, nor lets tools call it", async () => { + const read = tool("read"); + const bash = tool("bash"); + const probe = prober(["bash"]); + const setup = await setupWith([read, bash, probe.registration]); + setup.faux.setResponses([call("probe"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + // modelTools cannot bring back what tools disabled. + await root.configure({ tools: { remove: [bash] }, modelTools: [bash, probe.registration] }, context); + expect(await offered(root)).toEqual(["probe"]); + expect(await callable(root)).toEqual(["read", "probe"]); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(probe.received.get("bash")?.diagnostics[0]?.code).toBe("tool_unavailable"); + await harness.close(context); + }); + + it("answers a model-issued call of a tool it was not offered as unavailable", async () => { + const read = tool("read"); + const bash = tool("bash"); + const setup = await setupWith([read, bash]); + setup.faux.setResponses([call("bash"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ modelTools: [read] }, context); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const entries = await root.entries({}, 100, undefined, context); + const result = entries.items.find((entry) => entry.kind === "pi.tool-result"); + expect(JSON.stringify(result?.model)).toContain("Tool bash is not available"); + await harness.close(context); + }); + + it("applies addTools to modelTools too, so the model is offered the added tools", async () => { + const extra = tool("extra"); + const grow = tool("grow", { + execute: async () => ({ output: [], control: { addTools: ["extra"] } }), + }); + const setup = await setupWith([extra, grow]); + setup.faux.setResponses([call("grow"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await root.configure({ modelTools: [grow] }, context); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect((await harness.snapshot(AgentDoc, root.id, context))?.modelTools).toEqual(["grow", "extra"]); + expect(await offered(root)).toEqual(["grow", "extra"]); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-cancellation-barrier.test.ts b/packages/durable/test/harness-cancellation-barrier.test.ts new file mode 100644 index 000000000..5afd2a4f0 --- /dev/null +++ b/packages/durable/test/harness-cancellation-barrier.test.ts @@ -0,0 +1,223 @@ +import type { Context } from "@earendil-works/chord"; +import { createSession, defineTask, MemoryStorage, type StorageWrite, type TaskId } from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { context, flush } from "./session-support.ts"; +import { aborted, deferred, eventually, openTasks } from "./task-support.ts"; + +/** Runs until aborted; its abort handler ends it `aborted`. */ +const Dormant = defineTask({ + name: "test.dormant", + version: 1, + initial: () => ({ phase: "work" }), + phases: { + work: async (_task, runtime) => { + await aborted(runtime.signal); + }, + }, + abort: async (_task, runtime, abortContext) => { + await runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), abortContext); + }, +}); + +/** Holds every commit that abort-marks `target`, while set, until released. */ +class HoldMarks extends MemoryStorage { + target: TaskId | undefined; + readonly held = deferred(); + readonly release = deferred(); + override async commit(writes: readonly StorageWrite[], commitContext: Context) { + if ( + writes.some((write) => write.type === "task" && write.value.id === this.target && write.value.abortRequested) + ) { + this.held.resolve(); + await this.release.promise; + } + return super.commit(writes, commitContext); + } +} + +describe("work below a cancelled owner", () => { + it("does not start its next phase while the cascade that marks it has not committed", async () => { + const storage = new HoldMarks(); + const entered = deferred(); + const advance = deferred(); + let ranNext = false; + const Child = defineTask({ + name: "test.child", + version: 1, + initial: () => ({ phase: "first" }), + phases: { + first: async (_task, runtime, phaseContext) => { + entered.resolve(); + await advance.promise; + await runtime.commit(() => ({ status: "running", checkpoint: { phase: "next" } }), phaseContext); + }, + next: async (_task, runtime) => { + ranNext = true; + await aborted(runtime.signal); + }, + }, + abort: async (_task, runtime, abortContext) => { + await runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), abortContext); + }, + }); + const { harness } = await openTasks(storage, [Dormant, Child]); + const root = await harness.root(context); + const parent = await root.commit( + (tx) => tx.createTask(Dormant, null, { ownership: { kind: "conversation" } }), + context, + ); + const child = await root.commit( + (tx) => tx.createTask(Child, null, { ownership: { kind: "task", taskId: parent } }), + context, + ); + storage.target = child; + try { + harness.resume(); + await entered.promise; + const aborting = harness.abortTask(parent, context); + await storage.held.promise; + // The parent's request is durable, the cascade's mark of the child is not: its phase must not advance. + advance.resolve(); + for (let i = 0; i < 5; i++) await flush(); + expect(ranNext).toBe(false); + storage.release.resolve(); + expect((await harness.waitForTask(child, context)).state.outcome.status).toBe("aborted"); + await aborting; + expect(ranNext).toBe(false); + } finally { + advance.resolve(); + storage.release.resolve(); + await harness.close(context); + } + }); + + it("lets a failFast request in the same pass override a restart mark", async () => { + const storage = new MemoryStorage(); + const session = createSession(storage); + const conversation = await session.commit( + (tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), + context, + ); + const parent = await session.commit( + (tx) => tx.createTask(Dormant, null, { conversationId: conversation.id, ownership: { kind: "conversation" } }), + context, + ); + const failed = await session.commit( + (tx) => tx.createTask(Dormant, null, { ownership: { kind: "task", taskId: parent } }), + context, + ); + const child = await session.commit( + (tx) => tx.createTask(Dormant, null, { ownership: { kind: "task", taskId: parent } }), + context, + ); + // A restart-abandoned owner that waits failFast on a failed child and a live one. + const owner = (await storage.task(parent, context))!; + const failedRecord = (await storage.task(failed, context))!; + const waiting = { + status: "waiting", + checkpoint: { phase: "work" }, + on: [failed, child], + policy: "failFast", + } as const; + const failure = { status: "failed", error: { message: "failed" } } as const; + await storage.commit( + [ + { type: "task", value: { ...owner, abortRequested: true, abortReason: "restart", state: waiting } }, + { + type: "task", + value: { ...failedRecord, state: { status: "terminal", outcome: failure } } as typeof failedRecord, + }, + ], + context, + ); + // The child's definition is missing: a restart mark would keep it waiting, a request orphans it. + const { harness } = await openTasks(storage, []); + try { + harness.resume(); + await eventually(async () => (await harness.getTask(child, context))?.state.status === "terminal"); + expect((await harness.getTask(child, context))?.state).toMatchObject({ + outcome: { status: "orphaned", reason: "missing_task" }, + }); + } finally { + await harness.close(context); + } + }); + + it("upgrades restart marks down the tree when an abort is requested above them", async () => { + const storage = new MemoryStorage(); + const session = createSession(storage); + const conversation = await session.commit( + (tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), + context, + ); + const top = await session.commit( + (tx) => tx.createTask(Dormant, null, { conversationId: conversation.id, ownership: { kind: "conversation" } }), + context, + ); + const middle = await session.commit( + (tx) => tx.createTask(Dormant, null, { ownership: { kind: "task", taskId: top }, abandonOnRestart: true }), + context, + ); + const Missing = defineTask({ ...Dormant.definition, name: "test.missing" }); + const leaf = await session.commit( + (tx) => tx.createTask(Missing, null, { ownership: { kind: "task", taskId: middle } }), + context, + ); + // Only Dormant is registered: the leaf waits for its definition under the restart cascade. + const { harness } = await openTasks(storage, [Dormant]); + try { + harness.resume(); + await eventually(async () => (await harness.getTask(leaf, context))?.abortReason === "restart"); + await eventually(async () => { + const inspected = (await harness.inspect(context)).tasks.find((task) => task.record.id === leaf); + return inspected?.state.kind === "blocked"; + }); + expect((await harness.getTask(leaf, context))?.state.status).not.toBe("terminal"); + // A request on the top task reaches the leaf two levels down, which is then orphaned. + await harness.abortTask(top, context); + for (const [id, status] of [ + [leaf, "orphaned"], + [middle, "aborted"], + [top, "aborted"], + ] as const) { + expect((await harness.waitForTask(id, context)).state.outcome.status).toBe(status); + } + } finally { + await harness.close(context); + } + }); + + it("marks a flagged task that holds its outcome, aborting its owned work, and keeps the outcome", async () => { + const storage = new MemoryStorage(); + const session = createSession(storage); + const conversation = await session.commit( + (tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), + context, + ); + const holder = await session.commit( + (tx) => + tx.createTask(Dormant, null, { + conversationId: conversation.id, + ownership: { kind: "conversation" }, + abandonOnRestart: true, + }), + context, + ); + const child = await session.commit( + (tx) => tx.createTask(Dormant, null, { ownership: { kind: "task", taskId: holder } }), + context, + ); + const record = (await storage.task(holder, context))!; + const completing = { status: "completing", outcome: { status: "completed", result: null } } as const; + await storage.commit([{ type: "task", value: { ...record, state: completing } as typeof record }], context); + const { harness } = await openTasks(storage, [Dormant]); + try { + harness.resume(); + expect((await harness.waitForTask(child, context)).state.outcome.status).toBe("aborted"); + const settled = await harness.waitForTask(holder, context); + expect(settled).toMatchObject({ abortReason: "restart", state: { outcome: { status: "completed" } } }); + } finally { + await harness.close(context); + } + }); +}); diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index edd2d4bd9..28da7d753 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -25,7 +25,6 @@ import { LiveDoc, MemoryStorage, ProviderDoc, - StorageRejected, type TaskId, UsageDoc, watchEvents, @@ -455,7 +454,7 @@ describe("manual compaction", () => { name: "wait", description: "wait", parameters: Type.Object({}), - execute: async () => ({ content: [{ type: "text", text: "waited" }] }), + execute: async () => ({ output: [{ type: "text", text: "waited" }] }), }), ); await chat.root.configure({ tools: toolsNamed(chat.setup, "wait") }, context); @@ -869,7 +868,7 @@ describe("compaction outcomes", () => { name: "read", description: "read", parameters: Type.Object({}), - execute: async () => ({ content: [] }), + execute: async () => ({ output: [] }), }), ); await chat.root.configure({ tools: toolsNamed(chat.setup, "read") }, context); @@ -1268,7 +1267,7 @@ describe("compaction estimates and interactions", () => { execute: async () => { toolReached.resolve(); await toolGate.promise; - return { content: [{ type: "text", text: text("result", 200) }] }; + return { output: [{ type: "text", text: text("result", 200) }] }; }, }), ); @@ -2017,25 +2016,30 @@ describe("compaction edge cases", () => { await chat.harness.close(context); }); - it("leaves no usage, submission, summary, or outcome when the classifying commit is rejected", async () => { + it("leaves no usage, submission, or summary when the classifying commit fails, which fails the Harness", async () => { const storage = new ControlledStorage(); const chat = await open({ storage }); await history(chat); const usage = async () => (await chat.harness.snapshot(UsageDoc, chat.root.id, context))!.models["faux/faux-1"]; const before = await usage(); + const failure = new Error("disk gone"); chat.faux.summaries.push(async () => { - storage.failNextCommit(new StorageRejected("rejected")); + storage.failNextCommit(failure); return summary(); }); const id = await chat.root.compact(undefined, context); - const outcome = await result(chat, id); - expect(outcome.status).toBe("faulted"); - expect(await usage()).toEqual(before); - expect((await kinds(chat.root)).includes("pi.compaction")).toBe(false); - expect((await chat.harness.inspect(context)).submissions).toEqual([]); - expect(await storage.submissionByRequest(chat.root.id, `compaction:${id}`, context)).toBeUndefined(); - expect((await live(chat)).compactions).toBeUndefined(); + expect(await chat.harness.closed).toEqual({ reason: "failed", error: failure }); await chat.harness.close(context); + // Nothing of the failed commit landed: the task is still live, without usage, entry, or submission. + storage.reopen(); + expect(await storage.submissionByRequest(chat.root.id, `compaction:${id}`, context)).toBeUndefined(); + expect((await storage.task(id, context))?.state.status).not.toBe("terminal"); + const reopened = await openChat(storage, chat.setup); + expect((await reopened.harness.snapshot(UsageDoc, reopened.root.id, context))!.models["faux/faux-1"]).toEqual( + before, + ); + expect((await kinds(reopened.root)).includes("pi.compaction")).toBe(false); + await reopened.harness.close(context); }); }); diff --git a/packages/durable/test/harness-conversations.test.ts b/packages/durable/test/harness-conversations.test.ts index b92e46a17..ed5deba6e 100644 --- a/packages/durable/test/harness-conversations.test.ts +++ b/packages/durable/test/harness-conversations.test.ts @@ -449,11 +449,11 @@ describe("Harness lifecycle", () => { expect(again).not.toBe(root); expect(again.id).toBe(root.id); await harness.close(context); - await expect(harness.root(context)).rejects.toThrow("Harness is closed"); + await expect(harness.root(context)).rejects.toThrow("is closed"); await expect(harness.createConversation({ ownership: { kind: "ownerless" } }, context)).rejects.toThrow( - "Harness is closed", + "is closed", ); - await expect(harness.conversation(root.id, context)).rejects.toThrow("Harness is closed"); + await expect(harness.conversation(root.id, context)).rejects.toThrow("is closed"); }); it("forwards generic Session document APIs", async () => { diff --git a/packages/durable/test/harness-events.test.ts b/packages/durable/test/harness-events.test.ts index 9c11d4e22..22a5d2d75 100644 --- a/packages/durable/test/harness-events.test.ts +++ b/packages/durable/test/harness-events.test.ts @@ -191,7 +191,13 @@ describe("agent events", () => { await submission.wait(context); await drained(); const tool = events().filter((event) => event.type.startsWith("tool_execution")); - expect(tool[0]).toEqual({ type: "tool_execution_start", toolCallId: "c1", toolName: "print", args: { n: 1 } }); + expect(tool[0]).toEqual({ + type: "tool_execution_start", + toolCallId: "c1", + toolName: "print", + taskId: expect.any(Number), + args: { n: 1 }, + }); const end = tool.at(-1)!; expect(end).toMatchObject({ type: "tool_execution_end", toolCallId: "c1", entry: { kind: "pi.tool-result" } }); // As in the coding agent, the tool ends directly before its result message. diff --git a/packages/durable/test/harness-inbox.test.ts b/packages/durable/test/harness-inbox.test.ts index bd8b2a6f2..700b7c164 100644 --- a/packages/durable/test/harness-inbox.test.ts +++ b/packages/durable/test/harness-inbox.test.ts @@ -62,7 +62,7 @@ function gated(message: AssistantMessage): { step: FauxResponseStep; reached: Pr } /** Register a `hold` tool whose calls wait for `gate` and then return `result`. */ -function holdTool(setup: ChatSetup, gate: Deferred, result: ToolExecutionResult = { content: [] }): void { +function holdTool(setup: ChatSetup, gate: Deferred, result: ToolExecutionResult = { output: [] }): void { addTool( setup.registry, defineTool({ @@ -346,7 +346,7 @@ describe("inbox", () => { const setup = chatSetup(); const gate = deferred(); gate.resolve(); - holdTool(setup, gate, { content: [], control: { handoff: "continue here" } }); + holdTool(setup, gate, { output: [], control: { handoff: "continue here" } }); setup.faux.setResponses([HOLD]); const { harness, root } = await openChat(new MemoryStorage(), setup); const settled = await (await root.submit({ type: "input", content: "a" }, context)).wait(context); @@ -604,7 +604,7 @@ describe("inbox", () => { it("starts a queued follow-up after a terminating round", async () => { const setup = chatSetup(); const gate = deferred(); - holdTool(setup, gate, { content: [], control: { terminate: true } }); + holdTool(setup, gate, { output: [], control: { terminate: true } }); setup.faux.setResponses([HOLD, answer("follow-up")]); const { harness, root } = await openChat(new MemoryStorage(), setup); const input = await root.submit({ type: "input", content: "a" }, context); @@ -628,7 +628,7 @@ describe("inbox", () => { const setup = chatSetup(); const firstGate = deferred(); const secondGate = deferred(); - holdTool(setup, firstGate, { content: [], control: { handoff: "one" } }); + holdTool(setup, firstGate, { output: [], control: { handoff: "one" } }); addTool( setup.registry, defineTool({ @@ -637,7 +637,7 @@ describe("inbox", () => { parameters: Type.Object({}), execute: async () => { await secondGate.promise; - return { content: [], control: { handoff: "two" } }; + return { output: [], control: { handoff: "two" } }; }, }), ); @@ -782,7 +782,7 @@ describe("usage", () => { }; const gate = deferred(); gate.resolve(); - holdTool(setup, gate, { content: [], usage: spent }); + holdTool(setup, gate, { output: [], usage: spent }); setup.faux.setResponses([HOLD, answer("done"), answer("other")]); const { harness, root } = await openChat(new MemoryStorage(), setup); await (await root.submit({ type: "input", content: "a" }, context)).wait(context); diff --git a/packages/durable/test/harness-lifecycle.test.ts b/packages/durable/test/harness-lifecycle.test.ts index 425a5f81e..395e94ccb 100644 --- a/packages/durable/test/harness-lifecycle.test.ts +++ b/packages/durable/test/harness-lifecycle.test.ts @@ -102,7 +102,7 @@ describe("Harness open", () => { expect(await storageOpen(storage)).toBe(false); }); - it("reports a failing close and still rethrows the open error", async () => { + it("rethrows the open error and reports it once, though closing then fails too", async () => { class FailingClose extends ControlledStorage { override async close(closeContext: Context): Promise { await super.close(closeContext); @@ -119,7 +119,8 @@ describe("Harness open", () => { context, ); await expect(opening).rejects.toThrow("disk full"); - expect(reports).toEqual([new Error("close failed")]); + // The storage failure that failed open is the cause; the close that fails after it adds no report. + expect(reports).toEqual([new Error("disk full")]); }); }); @@ -167,7 +168,7 @@ describe("Harness close", () => { parameters: Type.Object({}), execute: async () => { if (where === "tool") await stubborn(); - return { content: [] }; + return { output: [] }; }, }), ); diff --git a/packages/durable/test/harness-live-deltas.test.ts b/packages/durable/test/harness-live-deltas.test.ts index 5cf68d464..488c1cd69 100644 --- a/packages/durable/test/harness-live-deltas.test.ts +++ b/packages/durable/test/harness-live-deltas.test.ts @@ -105,7 +105,7 @@ describe("pi.live deltas", () => { name: "noop", description: "noop", parameters: Type.Object({}), - execute: async () => ({ content: [] }), + execute: async () => ({ output: [] }), }), ); setup.faux.setResponses([ @@ -349,7 +349,7 @@ describe("pi.live deltas", () => { description: "bad", parameters: Type.Object({}), // Not strict JSON: the result commit throws and the scheduler faults the task. - execute: async () => ({ content: [], details: { fn: (() => 1) as never } }), + execute: async () => ({ output: [], details: { fn: (() => 1) as never } }), }), ); setup.faux.setResponses([ @@ -390,7 +390,7 @@ describe("pi.live deltas", () => { name: "noop", description: "noop", parameters: Type.Object({}), - execute: async () => ({ content: [] }), + execute: async () => ({ output: [] }), }), ); setup.faux.setResponses([ diff --git a/packages/durable/test/harness-nested-tools-restart.test.ts b/packages/durable/test/harness-nested-tools-restart.test.ts new file mode 100644 index 000000000..3212f1ae9 --- /dev/null +++ b/packages/durable/test/harness-nested-tools-restart.test.ts @@ -0,0 +1,631 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { Context } from "@earendil-works/chord"; +import { + type AssistantMessage, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type ToolResultMessage, + Type, +} from "@earendil-works/pi-ai"; +import { + type AgentEvent, + defineExtension, + defineTask, + defineTool, + type Harness, + LiveDoc, + type NestedToolExecutionResult, + type TaskId, + type ToolRegistration, + ToolResultEntry, + ToolTask, + type ToolTaskInput, + watchEvents, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; +import { addHooks, addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, deferred, eventually } from "./task-support.ts"; + +const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); + +function tool( + name: string, + execute: ToolRegistration["execute"], + extra: Partial> = {}, +): ToolRegistration { + return defineTool({ name, description: name, parameters: Parameters, execute, ...extra }); +} + +function call(name: string, id = "c1"): AssistantMessage { + return fauxAssistantMessage([fauxToolCall(name, {}, { id })], { stopReason: "toolUse" }); +} + +const DONE = fauxAssistantMessage([fauxText("done")]); + +const directories = new Set(); + +afterEach(async () => { + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +async function sqlitePath(): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-nested-restart-")); + directories.add(directory); + return join(directory, "session.sqlite"); +} + +/** Open the SQLite Harness without starting scheduling. */ +function open(path: string, setup: ChatSetup) { + return openNodeSqliteStorage(path).then((storage) => openChat(storage, setup)); +} + +async function nestedTasks(harness: Harness) { + const tasks = (await harness.commit((tx) => tx.scanTasks({ kind: ToolTask.definition.name }, 100), context)).items; + return tasks.filter((task) => (task.input as ToolTaskInput).kind === "nested"); +} + +function results(entries: Awaited>): ToolResultMessage[] { + return entries.filter((entry) => ToolResultEntry.is(entry)).map((entry) => entry.model![0] as ToolResultMessage); +} + +/** Collect tool end events from now on. */ +async function collectEnds(harness: Harness, conversationId: Parameters[0]) { + const ends: Extract[] = []; + const stream = await watchEvents(harness, conversationId, context); + stream.start(async (batch) => { + for (const event of batch) if (event.type === "tool_execution_end") ends.push(event); + }); + return { ends, stop: () => stream.stop() }; +} + +function codes(result: NestedToolExecutionResult | undefined): string[] { + return (result?.diagnostics ?? []).flatMap((diagnostic) => (diagnostic.code === undefined ? [] : [diagnostic.code])); +} + +describe("nested tool calls across a restart", () => { + it("abandons a replay-unsafe caller's unfinished nested calls before they run again", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const runs = { started: 0, gated: 0 }; + const running = deferred(); + const gated = deferred(); + // Replay-safe, so only the abandonment keeps it from rerunning. + addTool( + setup.registry, + tool( + "started", + async (_args, api, callContext) => { + runs.started++; + api.output("partial\n"); + await api.details({ step: 1 }, callContext); + running.resolve(); + return aborted(callContext.abortSignal!); + }, + { replay: "safe" }, + ), + ); + addTool( + setup.registry, + tool("gated", async () => { + runs.gated++; + return {}; + }), + ); + // Holds `gated` in its `call` phase, before intent, until the Harness closes. + addHooks(setup.registry, ToolTask, { + beforeTool: async (hookCall, _api, hookContext: Context) => { + if (hookCall.name !== "gated") return undefined; + gated.resolve(); + return aborted(hookContext.abortSignal!); + }, + }); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + const first = api.executeTool("started", {}, callContext); + const second = api.executeTool("gated", {}, callContext); + await Promise.all([first, second]); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await Promise.all([running.promise, gated.promise]); + await opened.harness.close(context); + + opened = await open(path, setup); + const { ends, stop } = await collectEnds(opened.harness, opened.root.id); + opened.harness.resume(); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + await opened.harness.waitForIdle(context); + await stop(); + expect(runs).toEqual({ started: 1, gated: 0 }); + const [started, gate] = await nestedTasks(opened.harness); + for (const task of [started, gate]) { + expect(task).toMatchObject({ abandonOnRestart: true, abortRequested: true, abortReason: "restart" }); + expect(task!.state).toMatchObject({ status: "terminal", outcome: { status: "aborted" } }); + } + const end = (callId: string) => ends.find((event) => event.toolCallId === callId)?.result; + expect(codes(end("c1/1"))).toEqual(["interrupted"]); + expect(end("c1/1")).toMatchObject({ isError: true, details: { step: 1 } }); + expect(end("c1/1")).not.toHaveProperty("structuredOutput"); + expect(codes(end("c1/2"))).toEqual(["abandoned"]); + // The caller reports the interruption. + const [result] = results(await allEntries(opened.root)); + expect(result).toMatchObject({ toolCallId: "c1", isError: true }); + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + }); + + it("abandons the work an abandoned nested call owns, which never runs a phase again", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const phases: string[] = []; + const aborts: (string | undefined)[] = []; + const started = deferred(); + const Worker = defineTask({ + name: "test.worker", + version: 1, + initial: () => ({ phase: "work" }), + phases: { + work: async (_task, runtime) => { + phases.push("work"); + started.resolve(); + await aborted(runtime.signal); + }, + }, + abort: async (task, runtime, abortContext) => { + aborts.push(task.abortReason); + await runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), abortContext); + }, + }); + setup.registry.install(defineExtension({ name: "worker", tasks: [Worker] })); + let worker: TaskId | undefined; + addTool( + setup.registry, + tool("spawn", async (_args, api, callContext) => { + worker = await api.createTask( + Worker, + null, + { ownership: { kind: "task", taskId: api.taskId } }, + callContext, + ); + await api.waitForTask(worker, callContext); + return {}; + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("spawn", {}, callContext); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + await opened.harness.close(context); + + opened = await open(path, setup); + opened.harness.resume(); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + await opened.harness.waitForIdle(context); + expect(phases).toEqual(["work"]); + // The cascade passed the restart reason on. + expect(aborts).toEqual(["restart"]); + expect((await opened.harness.getTask(worker!, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "aborted" }, + }); + await opened.harness.close(context); + }); + + it("keeps an abandoned task whose definition is missing waiting until it is installed, then runs its abort", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const aborts: (string | undefined)[] = []; + const started = deferred(); + const Worker = defineTask({ + name: "test.late", + version: 1, + initial: () => ({ phase: "work" }), + phases: { + work: async (_task, runtime) => { + started.resolve(); + await aborted(runtime.signal); + }, + }, + abort: async (task, runtime, abortContext) => { + aborts.push(task.abortReason); + await runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), abortContext); + }, + }); + const extension = defineExtension({ name: "late", tasks: [Worker] }); + setup.registry.install(extension); + let worker: TaskId | undefined; + addTool( + setup.registry, + tool("spawn", async (_args, api, callContext) => { + worker = await api.createTask( + Worker, + null, + { ownership: { kind: "task", taskId: api.taskId } }, + callContext, + ); + await api.waitForTask(worker, callContext); + return {}; + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("spawn", {}, callContext); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + await opened.harness.close(context); + + setup.registry.uninstall(extension); + opened = await open(path, setup); + opened.harness.resume(); + await eventually(async () => { + const inspected = (await opened.harness.inspect(context)).tasks.find((task) => task.record.id === worker); + return inspected?.state.kind === "blocked"; + }); + expect((await opened.harness.getTask(worker!, context))?.state.status).not.toBe("terminal"); + expect(aborts).toEqual([]); + setup.registry.install(extension); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(aborts).toEqual(["restart"]); + expect((await opened.harness.getTask(worker!, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "aborted" }, + }); + await opened.harness.close(context); + }); + + it("orphans an abandoned task whose definition is missing once an abort is requested", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const started = deferred(); + const Worker = defineTask({ + name: "test.gone", + version: 1, + initial: () => ({ phase: "work" }), + phases: { + work: async (_task, runtime) => { + started.resolve(); + await aborted(runtime.signal); + }, + }, + abort: async (_task, runtime, abortContext) => { + await runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), abortContext); + }, + }); + const extension = defineExtension({ name: "gone", tasks: [Worker] }); + setup.registry.install(extension); + let worker: TaskId | undefined; + addTool( + setup.registry, + tool("spawn", async (_args, api, callContext) => { + worker = await api.createTask( + Worker, + null, + { ownership: { kind: "task", taskId: api.taskId } }, + callContext, + ); + await api.waitForTask(worker, callContext); + return {}; + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("spawn", {}, callContext); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + await opened.harness.close(context); + + setup.registry.uninstall(extension); + opened = await open(path, setup); + opened.harness.resume(); + await eventually(async () => { + const record = await opened.harness.getTask(worker!, context); + return record?.abortReason === "restart"; + }); + await opened.root.abort(context); + expect((await opened.harness.getTask(worker!, context))?.state).toMatchObject({ + status: "terminal", + outcome: { status: "orphaned", reason: "missing_task" }, + }); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("unanswered"); + await opened.harness.close(context); + }); + + it("marks nothing when the reopened Harness never starts scheduling", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const running = deferred(); + addTool( + setup.registry, + tool("hang", async (_args, _api, callContext) => { + running.resolve(); + return aborted(callContext.abortSignal!); + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("hang", {}, callContext); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + let opened = await open(path, setup); + await opened.root.submit({ type: "input", content: "go" }, context); + await running.promise; + await opened.harness.close(context); + + // An inspection-only open: no resume, no submission. + opened = await open(path, setup); + const inspected = await opened.harness.inspect(context); + expect(inspected.scheduling).toBe("paused"); + await opened.harness.close(context); + + opened = await open(path, setup); + const [nested] = await nestedTasks(opened.harness); + expect(nested).toMatchObject({ abandonOnRestart: true, abortRequested: false }); + expect(nested!.state.status).toBe("pending"); + await opened.harness.close(context); + }); + + it("lets a replay-safe caller's unfinished nested calls recover and reattaches to them by key", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const runs = { safe: 0, unsafe: 0, caller: 0 }; + const running = { safe: deferred(), unsafe: deferred() }; + addTool( + setup.registry, + tool( + "safe", + async (_args, _api, callContext) => { + runs.safe++; + if (runs.safe === 1) { + running.safe.resolve(); + await aborted(callContext.abortSignal!); + } + return { output: [{ type: "text", text: `safe run ${runs.safe}` }] }; + }, + { replay: "safe" }, + ), + ); + addTool( + setup.registry, + tool("unsafe", async (_args, _api, callContext) => { + runs.unsafe++; + running.unsafe.resolve(); + return aborted(callContext.abortSignal!); + }), + ); + let results_: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + tool( + "batch", + async (_args, api, callContext) => { + runs.caller++; + results_ = await Promise.all([ + api.executeTool("safe", {}, callContext), + api.executeTool("unsafe", {}, callContext), + ]); + return {}; + }, + { replay: "safe" }, + ), + ); + setup.faux.setResponses([call("batch"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await Promise.all([running.safe.promise, running.unsafe.promise]); + await opened.harness.close(context); + + opened = await open(path, setup); + opened.harness.resume(); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(runs).toEqual({ safe: 2, unsafe: 1, caller: 2 }); + // Default keys follow call order, so the rerun found both calls. + expect(await nestedTasks(opened.harness)).toHaveLength(2); + for (const task of await nestedTasks(opened.harness)) { + expect(task.abandonOnRestart).toBeUndefined(); + expect(task.abortReason).toBeUndefined(); + } + expect(results_[0]?.structuredOutput).toBe("safe run 2"); + expect(codes(results_[1])).toEqual(["interrupted"]); + await opened.harness.close(context); + }); + + it("reattaches a replay-safe caller by default and explicit keys, and rejects a rerun that calls differently", async () => { + for (const order of ["same", "swapped"] as const) { + const path = await sqlitePath(); + const setup = chatSetup(); + const runs: string[] = []; + addTool( + setup.registry, + tool("echo", async (args) => { + runs.push(args.text ?? ""); + return { output: [{ type: "text", text: `echo ${args.text}` }] }; + }), + ); + const blocked = deferred(); + let callerRuns = 0; + let seen: string[] = []; + addTool( + setup.registry, + tool( + "batch", + async (_args, api, callContext) => { + callerRuns++; + const first = callerRuns === 2 && order === "swapped" ? "b" : "a"; + const one = await api.executeTool("echo", { text: first }, callContext); + const named = await api.executeTool("echo", { text: "named" }, callContext, { key: "named" }); + if (callerRuns === 1) { + blocked.resolve(); + await aborted(callContext.abortSignal!); + } + seen = [one, named].map((result) => result.structuredOutput as string); + return {}; + }, + { replay: "safe" }, + ), + ); + setup.faux.setResponses([call("batch"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await blocked.promise; + await opened.harness.close(context); + + opened = await open(path, setup); + opened.harness.resume(); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + const [result] = results(await allEntries(opened.root)); + if (order === "same") { + expect(runs).toEqual(["a", "named"]); + expect(seen).toEqual(["echo a", "echo named"]); + expect(result!.isError).toBe(false); + } else { + // Key 1 was made with other arguments: the rerun throws instead of reusing a different call's result. + expect(runs).toEqual(["a", "named"]); + expect(result!.isError).toBe(true); + expect(JSON.stringify(result!.content)).toContain("Nested call c1/1 was already made with another tool"); + } + await opened.harness.close(context); + } + }); + + it("abandons the child tasks a replay-unsafe tool created and awaited in memory", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const phases: string[] = []; + const started = deferred(); + const Worker = defineTask({ + name: "test.direct-worker", + version: 1, + initial: () => ({ phase: "work" }), + phases: { + work: async (_task, runtime) => { + phases.push("work"); + started.resolve(); + await aborted(runtime.signal); + }, + }, + abort: async (task, runtime, abortContext) => { + phases.push(`abort ${task.abortReason}`); + await runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), abortContext); + }, + }); + setup.registry.install(defineExtension({ name: "direct-worker", tasks: [Worker] })); + addTool( + setup.registry, + tool("work", async (_args, api, callContext) => { + const id = await api.createTask( + Worker, + null, + { ownership: { kind: "task", taskId: api.taskId } }, + callContext, + ); + await api.waitForTask(id, callContext); + return {}; + }), + ); + setup.faux.setResponses([call("work"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + await opened.harness.close(context); + + opened = await open(path, setup); + opened.harness.resume(); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(phases).toEqual(["work", "abort restart"]); + await opened.harness.close(context); + }); + + it("leaves background work under an abandoned nested call running, and settles without it", async () => { + const path = await sqlitePath(); + const setup = chatSetup(); + const started = deferred(); + const Background = defineTask({ + name: "test.background", + version: 1, + initial: () => ({ phase: "work" }), + phases: { + work: async (_task, runtime) => { + started.resolve(); + await aborted(runtime.signal); + }, + }, + abort: async (_task, runtime, abortContext) => { + await runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), abortContext); + }, + }); + setup.registry.install(defineExtension({ name: "background", tasks: [Background] })); + let background: TaskId | undefined; + addTool( + setup.registry, + tool("spawn", async (_args, api, callContext) => { + // A conversation the call owns, with background work in it, as a persistent subagent has. + background = await api.commit(async (tx) => { + const child = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); + return tx.createTask(Background, null, { + conversationId: child.id, + ownership: { kind: "conversation" }, + background: true, + }); + }, callContext); + return aborted(callContext.abortSignal!); + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("spawn", {}, callContext); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + let opened = await open(path, setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await started.promise; + await opened.harness.close(context); + + opened = await open(path, setup); + opened.harness.resume(); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + const [spawn] = await nestedTasks(opened.harness); + expect(spawn).toMatchObject({ abortReason: "restart", state: { status: "terminal" } }); + expect(await opened.harness.getTask(background!, context)).toMatchObject({ + abortRequested: false, + state: { status: "running" }, + }); + await opened.harness.abortTask(background!, context); + await opened.harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-nested-tools.test.ts b/packages/durable/test/harness-nested-tools.test.ts new file mode 100644 index 000000000..83cab2ae9 --- /dev/null +++ b/packages/durable/test/harness-nested-tools.test.ts @@ -0,0 +1,1270 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { JsonValue } from "@earendil-works/chord"; +import { withAbortSignal } from "@earendil-works/chord/context"; +import { + type AssistantMessage, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type ToolResultMessage, + Type, + type Usage, +} from "@earendil-works/pi-ai"; +import { + type AgentEvent, + AssistantEntry, + defineTool, + type EntryRecord, + type Harness, + LiveDoc, + MemoryStorage, + NestedResultDoc, + type NestedToolExecutionResult, + type TaskId, + type ToolHookCall, + type ToolRegistration, + ToolResultEntry, + ToolTask, + type ToolTaskInput, + type ToolTaskResult, + UsageDoc, + watchEvents, +} from "@earendil-works/pi-durable"; +import { afterEach, describe, expect, it } from "vitest"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; +import { addHooks, addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; +import { aborted, deferred, eventually } from "./task-support.ts"; + +const Parameters = Type.Object({ text: Type.Optional(Type.String()) }); +type Tool = ToolRegistration; + +function tool(name: string, execute: Tool["execute"], extra: Partial = {}): Tool { + return defineTool({ name, description: name, parameters: Parameters, execute, ...extra }); +} + +/** Returns `echo ` and counts its runs. */ +function echoTool(extra: Partial = {}) { + const state = { runs: 0 }; + const registration = tool( + "echo", + async (args) => { + state.runs++; + return { output: [{ type: "text", text: `echo ${args.text ?? ""}` }] }; + }, + extra, + ); + return { registration, state }; +} + +function call(name: string, args: Record = {}, id = "c1"): AssistantMessage { + return fauxAssistantMessage([fauxToolCall(name, args, { id })], { stopReason: "toolUse" }); +} + +const DONE = fauxAssistantMessage([fauxText("done")]); + +function results(entries: readonly EntryRecord[]): ToolResultMessage[] { + return entries.filter((entry) => ToolResultEntry.is(entry)).map((entry) => entry.model![0] as ToolResultMessage); +} + +/** The text of a result message, or the value a schema-less nested call gives programs. */ +function text(result: NestedToolExecutionResult | ToolResultMessage | undefined): string { + if (result === undefined) return ""; + if (!("role" in result)) return typeof result.structuredOutput === "string" ? result.structuredOutput : ""; + return result.content.map((item) => (item.type === "text" ? item.text : "")).join("|"); +} + +async function toolTasks(harness: Harness) { + return (await harness.commit((tx) => tx.scanTasks({ kind: ToolTask.definition.name }, 100), context)).items; +} + +async function runOnce(setup: ChatSetup, responses: AssistantMessage[]) { + setup.faux.setResponses(responses); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + return { harness, root, status: settled.status, entries: await allEntries(root) }; +} + +const directories = new Set(); + +afterEach(async () => { + for (const directory of directories) await rm(directory, { recursive: true, force: true }); + directories.clear(); +}); + +describe("nested tool calls", () => { + it("runs a nested call as its own tool task and returns its result to the caller, not the transcript", async () => { + const setup = chatSetup(); + const echo = echoTool(); + addTool(setup.registry, echo.registration); + let nested: NestedToolExecutionResult | undefined; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + nested = await api.executeTool("echo", { text: "a" }, callContext); + return { output: [{ type: "text", text: `got ${text(nested)}` }] }; + }), + ); + const { harness, root, status, entries } = await runOnce(setup, [call("batch"), DONE]); + expect(status).toBe("done"); + const [parent, child] = await toolTasks(harness); + expect(nested).toEqual({ + taskId: child!.id, + structuredOutput: "echo a", + isError: false, + diagnostics: [], + durationMs: expect.any(Number), + }); + // One result entry: the model-issued call's, which records nothing of its nested calls. + expect(results(entries).map((result) => [result.toolCallId, text(result)])).toEqual([["c1", "got echo a"]]); + expect(results(entries)[0]).not.toHaveProperty("nestedCalls"); + expect(parent!.input).toMatchObject({ kind: "model", callId: "c1" }); + expect(child!.owner).toBe(parent!.id); + expect(child!.input as ToolTaskInput).toEqual({ + kind: "nested", + parent: parent!.id, + parentCallId: "c1", + key: "1", + call: { type: "toolCall", id: "c1/1", name: "echo", arguments: { text: "a" } }, + }); + // The receipt stays small: the result lived in the caller's documents, which retired with it. + expect(child!.state).toEqual({ + status: "terminal", + outcome: { status: "completed", result: { kind: "nested" } }, + }); + expect(await harness.snapshot(NestedResultDoc, parent!.id, String(child!.id), context)).toBeUndefined(); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + it("gives each nested call its own task and call ID", async () => { + const setup = chatSetup(); + const seen: { taskId: TaskId; callId: string }[] = []; + addTool( + setup.registry, + tool("who", async (_args, api) => { + seen.push({ taskId: api.taskId, callId: api.callId }); + return {}; + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await Promise.all([ + api.executeTool("who", {}, callContext, { key: "a" }), + api.executeTool("who", {}, callContext, { key: "b" }), + ]); + return {}; + }), + ); + const { harness } = await runOnce(setup, [call("batch"), DONE]); + expect(seen.map((each) => each.callId).sort()).toEqual(["c1/a", "c1/b"]); + expect(new Set(seen.map((each) => each.taskId)).size).toBe(2); + await harness.close(context); + }); + + it("runs the tool hooks on nested calls, which see their parent", async () => { + const setup = chatSetup(); + const echo = echoTool(); + addTool(setup.registry, echo.registration); + const parents: ToolHookCall["parent"][] = []; + addHooks(setup.registry, ToolTask, { + beforeTool: (hookCall) => { + parents.push(hookCall.parent); + return hookCall.parent !== undefined && hookCall.arguments.text === "secret" + ? { block: "no secrets" } + : undefined; + }, + afterTool: (hookCall, result) => + hookCall.parent === undefined ? undefined : { ...result, output: [{ type: "text", text: "rewritten" }] }, + }); + const nested: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + nested.push(await api.executeTool("echo", { text: "secret" }, callContext)); + nested.push(await api.executeTool("echo", { text: "public" }, callContext)); + return {}; + }), + ); + const { harness } = await runOnce(setup, [call("batch"), DONE]); + const [parent] = await toolTasks(harness); + expect(parents).toEqual([undefined, { taskId: parent!.id, callId: "c1" }, { taskId: parent!.id, callId: "c1" }]); + expect(nested[0]).toMatchObject({ + isError: true, + diagnostics: [{ severity: "error", code: "blocked", message: "Tool call blocked: no secrets" }], + }); + expect(text(nested[1])).toBe("rewritten"); + expect(echo.state.runs).toBe(1); + await harness.close(context); + }); + + it("returns an error result for an unavailable tool and invalid arguments", async () => { + const setup = chatSetup(); + addTool(setup.registry, echoTool().registration); + const nested: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + nested.push(await api.executeTool("missing", {}, callContext)); + nested.push(await api.executeTool("echo", { text: { not: "a string" } }, callContext)); + return {}; + }), + ); + const { harness } = await runOnce(setup, [call("batch"), DONE]); + expect(nested.map((result) => [result.isError, result.diagnostics[0]?.code])).toEqual([ + [true, "tool_unavailable"], + [true, "invalid_arguments"], + ]); + await harness.close(context); + }); + + it("returns the call a key already names, and rejects reusing the key for another call", async () => { + const setup = chatSetup(); + const echo = echoTool(); + addTool(setup.registry, echo.registration); + let reused: NestedToolExecutionResult | undefined; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("echo", { text: "a" }, callContext, { key: "k" }); + reused = await api.executeTool("echo", { text: "a" }, callContext, { key: "k" }); + await api.executeTool("echo", { text: "b" }, callContext, { key: "k" }); + return {}; + }), + ); + const { harness, entries } = await runOnce(setup, [call("batch"), DONE]); + expect(echo.state.runs).toBe(1); + expect(text(reused)).toBe("echo a"); + expect(text(results(entries)[0])).toContain( + "Nested call c1/k was already made with another tool or other arguments", + ); + await harness.close(context); + }); + + it("matches a reused key's arguments regardless of key order", async () => { + const setup = chatSetup(); + let runs = 0; + addTool( + setup.registry, + defineTool({ + name: "pair", + description: "pair", + parameters: Type.Object({ a: Type.String(), b: Type.String() }), + execute: async (args) => { + runs++; + return { output: [{ type: "text", text: `${args.a}${args.b}` }] }; + }, + }), + ); + const nested: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + nested.push(await api.executeTool("pair", { a: "x", b: "y" }, callContext, { key: "k" })); + nested.push(await api.executeTool("pair", { b: "y", a: "x" }, callContext, { key: "k" })); + return {}; + }), + ); + const { harness } = await runOnce(setup, [call("batch"), DONE]); + expect(runs).toBe(1); + expect(nested.map((result) => text(result))).toEqual(["xy", "xy"]); + await harness.close(context); + }); + + it("aborts the nested calls a caller left running when it returns, keeping their details", async () => { + const setup = chatSetup(); + const running = deferred(); + addTool( + setup.registry, + tool("hang", async (_args, api, callContext) => { + api.output("partial\n"); + await api.details({ step: 1 }, callContext); + running.resolve(); + return aborted(callContext.abortSignal!); + }), + ); + let abandoned: Promise | undefined; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + abandoned = api.executeTool("hang", {}, callContext); + abandoned.catch(() => {}); + await running.promise; + return { output: [{ type: "text", text: "returned early" }] }; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const stream = await watchEvents(harness, root.id, context); + const ends: AgentEvent[] = []; + stream.start(async (batch) => { + ends.push(...batch.filter((event) => event.type === "tool_execution_end")); + }); + const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + expect(settled.status).toBe("done"); + await harness.waitForIdle(context); + await stream.stop(); + const [result] = results(await allEntries(root)); + expect(text(result)).toBe("returned early"); + const [, child] = await toolTasks(harness); + expect(child!.state).toMatchObject({ + status: "terminal", + outcome: { status: "aborted", result: { kind: "nested" } }, + }); + // The aborted call's result keeps its details; programs get no partial output, which only the model read. + const childEnd = ends.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1/1"); + expect(childEnd).toMatchObject({ result: { isError: true, details: { step: 1 } } }); + expect((childEnd as { result?: NestedToolExecutionResult }).result).not.toHaveProperty("structuredOutput"); + // The nested call ends, with its result, before its caller. + expect(ends.map((event) => [event.type === "tool_execution_end" && event.toolCallId, "result" in event])).toEqual( + [ + ["c1/1", true], + ["c1", false], + ], + ); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + it("records a nested call's usage under its tool", async () => { + const setup = chatSetup(); + const usage: Usage = { + input: 1, + output: 2, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 3, + cost: { input: 0.1, output: 0.2, cacheRead: 0, cacheWrite: 0, total: 0.3 }, + }; + addTool( + setup.registry, + tool("paid", async () => ({ usage })), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("paid", {}, callContext); + return {}; + }), + ); + const { harness, root } = await runOnce(setup, [call("batch"), DONE]); + expect((await harness.snapshot(UsageDoc, root.id, context))?.tools).toEqual({ paid: usage }); + await harness.close(context); + }); + + it("reports nested calls in pi.live.nestedTools and as tool events with their parent", async () => { + const setup = chatSetup(); + const release = deferred(); + const running = deferred(); + addTool( + setup.registry, + tool("slow", async (_args, api, callContext) => { + api.output("working\n"); + await api.details({ step: 1 }, callContext); + running.resolve(); + await release.promise; + return { output: [{ type: "text", text: "slow done" }] }; + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + const result = await api.executeTool("slow", {}, callContext); + return { output: [{ type: "text", text: text(result) }] }; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const stream = await watchEvents(harness, root.id, context); + expect(stream.snapshot.nestedTools).toEqual([]); + const events: AgentEvent[] = []; + stream.start(async (batch) => { + events.push(...batch); + }); + const submission = await root.submit({ type: "input", content: "go" }, context); + await running.promise; + const live = await harness.snapshot(LiveDoc, root.id, context); + expect(live?.nestedTools).toEqual([ + { + callId: "c1/1", + parentCallId: "c1", + parentTaskId: expect.any(Number), + name: "slow", + taskId: expect.any(Number), + arguments: {}, + status: "running", + output: "working\n", + details: { step: 1 }, + }, + ]); + release.resolve(); + await submission.wait(context); + await harness.waitForIdle(context); + await stream.stop(); + const tools = events.flatMap((event) => + event.type === "tool_execution_start" || event.type === "tool_execution_end" + ? [[event.type, event.toolCallId, event.parentToolCallId]] + : [], + ); + expect(tools).toEqual([ + ["tool_execution_start", "c1", undefined], + ["tool_execution_start", "c1/1", "c1"], + ["tool_execution_end", "c1/1", "c1"], + ["tool_execution_end", "c1", undefined], + ]); + const nestedEnd = events.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1/1"); + expect(nestedEnd).toMatchObject({ result: { structuredOutput: "slow done", isError: false } }); + // Events name the call's task and, for a nested call, the calling one. + const [parent, child] = await toolTasks(harness); + const parentEnd = events.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1"); + expect(nestedEnd).toMatchObject({ taskId: child!.id, parentTaskId: parent!.id }); + expect(parentEnd).toMatchObject({ taskId: parent!.id }); + expect(parentEnd).not.toHaveProperty("parentTaskId"); + expect( + events.some( + (event) => + event.type === "tool_execution_update" && event.toolCallId === "c1/1" && event.parentToolCallId === "c1", + ), + ).toBe(true); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + it("aborts nested calls with their caller", async () => { + const setup = chatSetup(); + const running = deferred(); + addTool( + setup.registry, + tool("hang", async (_args, _api, callContext) => { + running.resolve(); + return aborted(callContext.abortSignal!); + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("hang", {}, callContext); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await running.promise; + await root.abort(context); + expect((await submission.wait(context)).status).toBe("unanswered"); + const [parent, child] = await toolTasks(harness); + const outcome = (record: typeof parent) => + record!.state.status === "terminal" ? record!.state.outcome : undefined; + expect(outcome(child)).toMatchObject({ status: "aborted", result: { kind: "nested" } }); + expect(outcome(parent)).toMatchObject({ status: "aborted", result: { kind: "model" } }); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + it("aborts a running nested call when its caller throws", async () => { + const setup = chatSetup(); + const running = deferred(); + addTool( + setup.registry, + tool("hang", async (_args, _api, callContext) => { + running.resolve(); + return aborted(callContext.abortSignal!); + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + void api.executeTool("hang", {}, callContext).catch(() => {}); + await running.promise; + throw new Error("caller failed"); + }), + ); + const { harness, status, entries } = await runOnce(setup, [call("batch"), DONE]); + expect(status).toBe("done"); + expect(text(results(entries)[0])).toContain("caller failed"); + const [, child] = await toolTasks(harness); + expect(child!.state).toMatchObject({ status: "terminal", outcome: { status: "aborted" } }); + await harness.close(context); + }); + + it("reattaches a replay-safe caller to the nested calls it made before a restart", async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-nested-")); + directories.add(directory); + const path = join(directory, "session.sqlite"); + const setup = chatSetup(); + const echo = echoTool(); + addTool(setup.registry, echo.registration); + const blocked = deferred(); + let runs = 0; + addTool( + setup.registry, + tool( + "batch", + async (_args, api, callContext) => { + runs++; + const first = await api.executeTool("echo", { text: "a" }, callContext); + if (runs === 1) { + blocked.resolve(); + await aborted(callContext.abortSignal!); + } + // A call new to the rerun: its slot, after the reattached one's, is found and finishes. + const second = await api.executeTool("echo", { text: "b" }, callContext); + return { output: [{ type: "text", text: `run ${runs}: ${text(first)}, ${text(second)}` }] }; + }, + { replay: "safe" }, + ), + ); + setup.faux.setResponses([call("batch"), DONE]); + let opened = await openChat(await openNodeSqliteStorage(path), setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await blocked.promise; + await opened.harness.close(context); + + opened = await openChat(await openNodeSqliteStorage(path), setup); + opened.harness.resume(); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + expect(runs).toBe(2); + expect(echo.state.runs).toBe(2); + expect(text(results(await allEntries(opened.root))[0])).toBe("run 2: echo a, echo b"); + const tasks = await toolTasks(opened.harness); + expect(tasks.filter((task) => (task.input as ToolTaskInput).kind === "nested")).toHaveLength(2); + const outcome = tasks[1]!.state.status === "terminal" ? tasks[1]!.state.outcome : undefined; + expect((outcome?.result as ToolTaskResult | undefined)?.kind).toBe("nested"); + await opened.harness.close(context); + }); + + it("stops only the wait when a caller cancels it; the call runs on and its key returns its result", async () => { + const setup = chatSetup(); + const cancel = new AbortController(); + const release = deferred(); + const attempts: string[] = []; + let retry: NestedToolExecutionResult | undefined; + addTool( + setup.registry, + tool("slow", async () => { + cancel.abort(new Error("cancelled")); + await release.promise; + return { output: [{ type: "text", text: "slow done" }] }; + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + const cancelled = withAbortSignal(cancel.signal, callContext); + await api.executeTool("slow", {}, cancelled, { key: "same" }).then( + () => attempts.push("first done"), + () => attempts.push("first cancelled"), + ); + release.resolve(); + retry = await api.executeTool("slow", {}, callContext, { key: "same" }); + return {}; + }), + ); + const { harness } = await runOnce(setup, [call("batch"), DONE]); + expect(attempts).toEqual(["first cancelled"]); + expect(retry).toMatchObject({ isError: false, structuredOutput: "slow done" }); + expect(await toolTasks(harness)).toHaveLength(2); + await harness.close(context); + }); + + it("admits one nested call for a key used by concurrent calls", async () => { + const setup = chatSetup(); + const echo = echoTool(); + addTool(setup.registry, echo.registration); + let both: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + both = await Promise.all([ + api.executeTool("echo", { text: "a" }, callContext, { key: "k" }), + api.executeTool("echo", { text: "a" }, callContext, { key: "k" }), + ]); + return {}; + }), + ); + const { harness } = await runOnce(setup, [call("batch"), DONE]); + expect(echo.state.runs).toBe(1); + expect(both.map((result) => text(result))).toEqual(["echo a", "echo a"]); + expect(await toolTasks(harness)).toHaveLength(2); + await harness.close(context); + }); + + it("aborts left-running nested calls of a caller interrupted while it cleaned them up", async () => { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-nested-")); + directories.add(directory); + const path = join(directory, "session.sqlite"); + const setup = chatSetup(); + const running = deferred(); + const cleaning = deferred(); + const release = deferred(); + let hangRuns = 0; + addTool( + setup.registry, + tool("hang", async (_args, api, callContext) => { + hangRuns++; + api.output("partial\n"); + await api.details({ step: 1 }, callContext); + running.resolve(); + await aborted(callContext.abortSignal!).catch(() => {}); + // The caller's cleanup waits for this abort; hold it there until the Harness closes. + cleaning.resolve(); + await release.promise; + throw new Error("aborted"); + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + void api.executeTool("hang", {}, callContext).catch(() => {}); + await running.promise; + return { output: [{ type: "text", text: "returned early" }] }; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + let opened = await openChat(await openNodeSqliteStorage(path), setup); + const id = (await opened.root.submit({ type: "input", content: "go" }, context)).id; + await cleaning.promise; + const closed = opened.harness.close(context); + release.resolve(); + await closed; + + opened = await openChat(await openNodeSqliteStorage(path), setup); + opened.harness.resume(); + expect((await (await opened.harness.submission(id, context))!.wait(context)).status).toBe("done"); + await opened.harness.waitForIdle(context); + // The caller is replay-unsafe: it reports the interruption, and its cleanup aborts the nested call. + const [result] = results(await allEntries(opened.root)); + expect(result).toMatchObject({ toolCallId: "c1", isError: true }); + expect(text(result)).toContain("Tool batch was interrupted"); + const [, child] = await toolTasks(opened.harness); + expect(child!.state).toMatchObject({ status: "terminal", outcome: { status: "aborted" } }); + expect(hangRuns).toBe(1); + expect(await opened.harness.snapshot(LiveDoc, opened.root.id, context)).toEqual({}); + await opened.harness.close(context); + }); + + it("aborts a nested call whose admission was still committing when its caller returned", async () => { + const setup = chatSetup(); + addTool(setup.registry, echoTool().registration); + addTool( + setup.registry, + tool("hang", async (_args, _api, callContext) => aborted(callContext.abortSignal!)), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("echo", { text: "warm" }, callContext, { key: "warm" }); + void api.executeTool("hang", {}, callContext, { key: "late" }).catch(() => {}); + return {}; + }), + ); + const { harness, root, status } = await runOnce(setup, [call("batch"), DONE]); + expect(status).toBe("done"); + const late = (await toolTasks(harness)).find((task) => { + const input = task.input as ToolTaskInput; + return input.kind === "nested" && input.call.name === "hang"; + }); + expect(late!.state).toMatchObject({ status: "terminal", outcome: { status: "aborted" } }); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + it("aborts every left-running nested call before waiting for any", async () => { + const setup = chatSetup(); + const bAborted = deferred(); + const started = { a: deferred(), b: deferred() }; + // A only stops once B was told to stop. + addTool( + setup.registry, + tool("a", async () => { + started.a.resolve(); + await bAborted.promise; + return {}; + }), + ); + addTool( + setup.registry, + tool("b", async (_args, _api, callContext) => { + started.b.resolve(); + await aborted(callContext.abortSignal!).catch(() => {}); + bAborted.resolve(); + throw new Error("aborted"); + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + void api.executeTool("a", {}, callContext, { key: "a" }).catch(() => {}); + void api.executeTool("b", {}, callContext, { key: "b" }).catch(() => {}); + await Promise.all([started.a.promise, started.b.promise]); + return {}; + }), + ); + const { harness, status } = await runOnce(setup, [call("batch"), DONE]); + expect(status).toBe("done"); + const nested = (await toolTasks(harness)).filter((task) => (task.input as ToolTaskInput).kind === "nested"); + expect(nested.map((task) => task.state.status)).toEqual(["terminal", "terminal"]); + await harness.close(context); + }); + + it("rejects explicit keys that are empty, have a slash, are __proto__, or are positive integers", async () => { + const setup = chatSetup(); + addTool(setup.registry, echoTool().registration); + const errors: string[] = []; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + for (const key of ["", "a/b", "__proto__", "7"]) { + await api + .executeTool("echo", {}, callContext, { key }) + .catch((error: Error) => errors.push(error.message)); + } + return {}; + }), + ); + const { harness } = await runOnce(setup, [call("batch"), DONE]); + const rule = 'must be non-empty, without "/", not "__proto__", and not a positive integer'; + expect(errors).toEqual([ + `Nested call key "" ${rule}`, + `Nested call key "a/b" ${rule}`, + `Nested call key "__proto__" ${rule}`, + `Nested call key "7" ${rule}`, + ]); + await harness.close(context); + }); + + it("runs nested calls of nested calls, and ends them children first", async () => { + const setup = chatSetup(); + const echo = echoTool(); + addTool(setup.registry, echo.registration); + addTool( + setup.registry, + tool("inner", async (_args, api, callContext) => { + const result = await api.executeTool("echo", { text: "deep" }, callContext, { key: "x" }); + return { output: [{ type: "text", text: text(result) }] }; + }), + ); + addTool( + setup.registry, + tool("outer", async (_args, api, callContext) => { + const result = await api.executeTool("inner", {}, callContext); + return { output: [{ type: "text", text: text(result) }] }; + }), + ); + setup.faux.setResponses([call("outer"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const stream = await watchEvents(harness, root.id, context); + const tools: (string | undefined)[][] = []; + stream.start(async (batch) => { + for (const event of batch) { + if (event.type === "tool_execution_start" || event.type === "tool_execution_end") { + tools.push([event.type, event.toolCallId, event.parentToolCallId]); + } + } + }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await harness.waitForIdle(context); + await stream.stop(); + expect(text(results(await allEntries(root))[0])).toBe("echo deep"); + const [outer, inner, deep] = await toolTasks(harness); + expect([inner!.owner, deep!.owner]).toEqual([outer!.id, inner!.id]); + expect(tools).toEqual([ + ["tool_execution_start", "c1", undefined], + ["tool_execution_start", "c1/1", "c1"], + ["tool_execution_start", "c1/1/x", "c1/1"], + ["tool_execution_end", "c1/1/x", "c1/1"], + ["tool_execution_end", "c1/1", "c1"], + ["tool_execution_end", "c1", undefined], + ]); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + it("reports the output of several running nested calls, and shows them to a late subscriber", async () => { + const setup = chatSetup(); + const release = deferred(); + const running = { one: deferred(), two: deferred() }; + const wrote = { one: deferred(), two: deferred() }; + addTool( + setup.registry, + tool("slow", async (args, api, callContext) => { + const name = args.text as "one" | "two"; + running[name].resolve(); + await wrote[name].promise; + api.output(`${name}\n`); + await api.details({ name }, callContext); + await release.promise; + return {}; + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await Promise.all([ + api.executeTool("slow", { text: "one" }, callContext), + api.executeTool("slow", { text: "two" }, callContext), + ]); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await Promise.all([running.one.promise, running.two.promise]); + const stream = await watchEvents(harness, root.id, context); + expect(stream.snapshot.nestedTools.map((slot) => [slot.callId, slot.status])).toEqual([ + ["c1/1", "running"], + ["c1/2", "running"], + ]); + const updates: [string, unknown][] = []; + stream.start(async (batch) => { + for (const event of batch) { + if (event.type === "tool_execution_update" && event.output !== undefined) { + updates.push([event.toolCallId, event.output]); + } + } + }); + // The second slot writes first, so a wrong slot index in the update path would show. + wrote.two.resolve(); + await eventually(() => updates.length === 1); + wrote.one.resolve(); + await eventually(() => updates.length === 2); + release.resolve(); + await submission.wait(context); + await stream.stop(); + expect(updates).toEqual([ + ["c1/2", { set: "two\n" }], + ["c1/1", { set: "one\n" }], + ]); + await harness.close(context); + }); + + it("commits only the status of a nested call made with progress: false, and keeps its output in the result", async () => { + const setup = chatSetup(); + const running = deferred(); + const release = deferred(); + addTool( + setup.registry, + tool("chatty", async (_args, api, callContext) => { + api.output("line\n"); + api.diagnostic({ severity: "info", message: "note" }); + await api.details({ step: 1 }, callContext); + running.resolve(); + await release.promise; + return {}; + }), + ); + let result: NestedToolExecutionResult | undefined; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + result = await api.executeTool("chatty", {}, callContext, { progress: false }); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + // `details()` resolved before this, so a progress commit would have landed already. + await running.promise; + const [slot] = (await harness.snapshot(LiveDoc, root.id, context))!.nestedTools!; + expect(slot).toEqual({ + callId: "c1/1", + parentCallId: "c1", + parentTaskId: expect.any(Number), + name: "chatty", + taskId: expect.any(Number), + arguments: {}, + status: "running", + }); + release.resolve(); + await submission.wait(context); + expect(result).toMatchObject({ + structuredOutput: "line\n", + details: { step: 1 }, + diagnostics: [{ severity: "info", message: "note" }], + }); + await harness.close(context); + }); + + it("loses the running output of a nested call made with progress: false when it is aborted", async () => { + const setup = chatSetup(); + const running = deferred(); + addTool( + setup.registry, + tool("chatty", async (_args, api, callContext) => { + api.output("line\n"); + api.diagnostic({ severity: "info", message: "note" }); + await api.details({ step: 1 }, callContext); + running.resolve(); + return aborted(callContext.abortSignal!); + }), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + void api.executeTool("chatty", {}, callContext, { progress: false }).catch(() => {}); + await running.promise; + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const stream = await watchEvents(harness, root.id, context); + const ends: AgentEvent[] = []; + stream.start(async (batch) => { + ends.push(...batch.filter((event) => event.type === "tool_execution_end")); + }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await harness.waitForIdle(context); + await stream.stop(); + // The abort's result is built from the slot, which never got the output. + const end = ends.find((event) => event.type === "tool_execution_end" && event.toolCallId === "c1/1"); + const result = (end as { result?: NestedToolExecutionResult }).result!; + expect(result).toMatchObject({ isError: true }); + expect(result).not.toHaveProperty("structuredOutput"); + expect(result).not.toHaveProperty("details"); + expect(result.diagnostics.map((diagnostic) => diagnostic.code)).toEqual(["aborted"]); + await harness.close(context); + }); + + it("shows a nested call's arguments in its slot: as made, then, unbounded, as it runs with them", async () => { + const setup = chatSetup(); + const hookEntered = deferred(); + const hookRelease = deferred(); + const running = deferred(); + const release = deferred(); + const large = "x".repeat(64 * 1024); + addTool( + setup.registry, + tool("slow", async () => { + running.resolve(); + await release.promise; + return {}; + }), + ); + addHooks(setup.registry, ToolTask, { + beforeTool: async (hookCall) => { + if (hookCall.parent === undefined) return undefined; + hookEntered.resolve(); + await hookRelease.promise; + return { arguments: { text: large } }; + }, + }); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("slow", { text: "asked" }, callContext); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const stream = await watchEvents(harness, root.id, context); + const starts: AgentEvent[] = []; + stream.start(async (batch) => { + starts.push(...batch.filter((event) => event.type === "tool_execution_start")); + }); + const submission = await root.submit({ type: "input", content: "go" }, context); + const slot = async () => (await harness.snapshot(LiveDoc, root.id, context))!.nestedTools![0]!; + await hookEntered.promise; + expect(await slot()).toMatchObject({ status: "pending", arguments: { text: "asked" } }); + hookRelease.resolve(); + await running.promise; + expect(await slot()).toMatchObject({ status: "running", arguments: { text: large } }); + release.resolve(); + await submission.wait(context); + await harness.waitForIdle(context); + await stream.stop(); + // The start event carries the arguments the call runs with; the task input keeps the ones it was made with. + const start = starts.find((event) => event.type === "tool_execution_start" && event.toolCallId === "c1/1"); + expect(start).toMatchObject({ args: { text: large } }); + const nested = (await toolTasks(harness)).find((task) => (task.input as ToolTaskInput).kind === "nested"); + expect(nested!.input as ToolTaskInput).toMatchObject({ call: { arguments: { text: "asked" } } }); + await harness.close(context); + }); + + it("runs a nested call whose repaired arguments set an optional property to undefined", async () => { + const setup = chatSetup(); + const seen: unknown[] = []; + addTool( + setup.registry, + tool( + "repaired", + async (args) => { + seen.push(args); + return {}; + }, + { prepareArguments: () => ({ text: undefined }) }, + ), + ); + let result: NestedToolExecutionResult | undefined; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + result = await api.executeTool("repaired", { text: "x" }, callContext); + return {}; + }), + ); + const { harness } = await runOnce(setup, [call("batch"), DONE]); + expect(result).toMatchObject({ isError: false }); + expect(seen).toEqual([{}]); + await harness.close(context); + }); + + it("finds each nested call's slot after slots in the middle were removed", async () => { + const setup = chatSetup(); + addTool(setup.registry, echoTool().registration); + addTool( + setup.registry, + tool("inner", async (_args, api, callContext) => { + await api.executeTool("echo", { text: "m1" }, callContext); + await api.executeTool("echo", { text: "m2" }, callContext); + return {}; + }), + ); + const before = deferred(); + const checked = deferred(); + let slots: { callId: string; status: string; summary?: unknown }[] = []; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("echo", { text: "n1" }, callContext); + // n2 makes m1 and m2 and settles, which removes them from between n2 and n3. + await api.executeTool("inner", {}, callContext); + await api.executeTool("echo", { text: "n3" }, callContext); + await api.executeTool("echo", { text: "n4" }, callContext); + const live = await api.snapshot(LiveDoc, api.conversationId, callContext); + slots = (live?.nestedTools ?? []).map(({ callId, status, summary }) => ({ callId, status, summary })); + before.resolve(); + await checked.promise; + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await before.promise; + const done = { status: "done", summary: { isError: false, durationMs: expect.any(Number) } }; + expect(slots).toEqual(["c1/1", "c1/2", "c1/3", "c1/4"].map((callId) => ({ callId, ...done }))); + checked.resolve(); + await submission.wait(context); + await harness.close(context); + }); + + it("gives callers a result with its task ID and only diagnostics for a nested call the scheduler faulted", async () => { + const setup = chatSetup(); + // Details that are not JSON make the result commit throw, so the task faults and stores no result. + addTool( + setup.registry, + tool("echo", async () => ({ details: { count: 1n } as unknown as JsonValue })), + ); + let result: NestedToolExecutionResult | undefined; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + result = await api.executeTool("echo", { text: "a" }, callContext); + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const [, child] = await toolTasks(harness); + expect(child!.state).toMatchObject({ status: "terminal", outcome: { status: "faulted" } }); + expect(result).toEqual({ + taskId: child!.id, + isError: true, + diagnostics: [{ severity: "error", code: "faulted", message: expect.stringContaining("Tool echo failed") }], + }); + await harness.close(context); + }); + + it("gives callers what a schema-less tool streamed before it threw", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + tool("fails", async (_args, api) => { + api.output("half way\n"); + throw new Error("broke"); + }), + ); + let result: NestedToolExecutionResult | undefined; + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + result = await api.executeTool("fails", {}, callContext); + return {}; + }), + ); + const { harness } = await runOnce(setup, [call("batch"), DONE]); + expect(result).toMatchObject({ + isError: true, + structuredOutput: "half way\n", + diagnostics: [{ code: "tool_error", message: "broke" }], + }); + await harness.close(context); + }); + + it("shows a finished nested call as a done slot with its summary until its caller settles", async () => { + const setup = chatSetup(); + const release = deferred(); + const checked = deferred(); + // Diagnostics win over output for the error text, which is cut to 500 characters. + addTool( + setup.registry, + tool("verbose", async () => ({ + output: [{ type: "text", text: "ignored" }], + isError: true, + diagnostics: [{ severity: "error", message: "e".repeat(600) }], + })), + ); + const usage: Usage = { + input: 1, + output: 2, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 3, + cost: { input: 0.1, output: 0.2, cacheRead: 0, cacheWrite: 0, total: 0.3 }, + }; + addTool(setup.registry, echoTool().registration); + addTool( + setup.registry, + tool("paid", async () => ({ usage })), + ); + // An error with mixed content: programs get the list, the summary its text. + addTool( + setup.registry, + tool("broken", async () => ({ + output: [ + { type: "text", text: "bad image" }, + { type: "image", data: "AAAA", mimeType: "image/png" }, + ], + isError: true, + })), + ); + addTool( + setup.registry, + tool("batch", async (_args, api, callContext) => { + await api.executeTool("echo", { text: "a" }, callContext); + await api.executeTool("missing", {}, callContext); + await api.executeTool("paid", {}, callContext); + await api.executeTool("broken", {}, callContext); + await api.executeTool("verbose", {}, callContext); + checked.resolve(); + await release.promise; + return {}; + }), + ); + setup.faux.setResponses([call("batch"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + const submission = await root.submit({ type: "input", content: "go" }, context); + await checked.promise; + const slots = (await harness.snapshot(LiveDoc, root.id, context))!.nestedTools!; + expect(slots.map(({ callId, status, summary }) => ({ callId, status, summary }))).toEqual([ + { callId: "c1/1", status: "done", summary: { isError: false, durationMs: expect.any(Number) } }, + { + callId: "c1/2", + status: "done", + summary: { isError: true, error: "Tool missing is not available" }, + }, + { callId: "c1/3", status: "done", summary: { isError: false, durationMs: expect.any(Number), usage } }, + { + callId: "c1/4", + status: "done", + summary: { isError: true, durationMs: expect.any(Number), error: "bad image" }, + }, + { + callId: "c1/5", + status: "done", + summary: { isError: true, durationMs: expect.any(Number), error: "e".repeat(500) }, + }, + ]); + release.resolve(); + await submission.wait(context); + expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); + await harness.close(context); + }); + + it("migrates version 1 tool task input to a model-issued call", () => { + const migrate = ToolTask.definition.migrate!; + expect(migrate({ assistant: 3, callId: "c1" }, { phase: "call" }, 1)).toEqual({ + input: { kind: "model", assistant: 3, callId: "c1" }, + checkpoint: { phase: "call" }, + }); + }); +}); + +describe("tool task version 1 records", () => { + /** + * Store a live tool task for call `c1` of an assistant entry, close, and rewrite its record as version 1 stored it: + * `{ assistant, callId }` input at version 1, with `checkpoint`. Returns the storage path and the task ID. + */ + async function storeVersion1(setup: ChatSetup, checkpoint: JsonValue): Promise<{ path: string; id: TaskId }> { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-tool-v1-")); + directories.add(directory); + const path = join(directory, "session.sqlite"); + // No submission, so the Harness never starts scheduling and the task stays pending. + const { harness, root } = await openChat(await openNodeSqliteStorage(path), setup); + const id = await root.commit(async (tx) => { + const entry = await tx.appendEntry(AssistantEntry, root.id, { model: [call("echo", { text: "old" })] }); + return tx.createTask( + ToolTask, + { kind: "model", assistant: entry.id, callId: "c1" }, + { ownership: { kind: "conversation" } }, + ); + }, context); + await harness.close(context); + const storage = await openNodeSqliteStorage(path); + const record = (await storage.task(id, context))!; + const { kind: _kind, ...input } = record.input as Extract; + const state = { ...record.state, checkpoint } as typeof record.state; + await storage.commit( + [{ type: "task", value: { ...record, version: 1, input, state } as typeof record }], + context, + ); + await storage.close(context); + return { path, id }; + } + + it("migrates a pending version 1 task and runs it as a model-issued call", async () => { + const setup = chatSetup(); + const echo = echoTool(); + addTool(setup.registry, echo.registration); + const { path, id } = await storeVersion1(setup, { phase: "call" }); + const { harness, root } = await openChat(await openNodeSqliteStorage(path), setup); + expect((await harness.getTask(id, context))?.version).toBe(1); + harness.resume(); + const settled = await harness.waitForTask(id, context); + expect(settled.version).toBe(2); + expect(settled.input).toMatchObject({ kind: "model", callId: "c1" }); + expect(settled.state.outcome).toMatchObject({ status: "completed", result: { kind: "model" } }); + expect(echo.state.runs).toBe(1); + const [result] = results(await allEntries(root)); + expect([result!.toolCallId, text(result)]).toEqual(["c1", "echo old"]); + await harness.close(context); + }); + + it("migrates a version 1 task interrupted after intent and settles it interrupted", async () => { + const setup = chatSetup(); + const echo = echoTool(); + addTool(setup.registry, echo.registration); + const intent = { phase: "execute", arguments: { text: "old" }, replay: "unsafe" }; + const { path, id } = await storeVersion1(setup, intent); + const { harness, root } = await openChat(await openNodeSqliteStorage(path), setup); + harness.resume(); + const settled = await harness.waitForTask(id, context); + expect(settled.version).toBe(2); + expect(settled.state.outcome).toMatchObject({ status: "failed", result: { kind: "model" } }); + expect(echo.state.runs).toBe(0); + const [result] = results(await allEntries(root)); + expect(result).toMatchObject({ toolCallId: "c1", isError: true }); + expect(text(result)).toContain("Tool echo was interrupted"); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-ownership.test.ts b/packages/durable/test/harness-ownership.test.ts index c46b51a29..389dfbe20 100644 --- a/packages/durable/test/harness-ownership.test.ts +++ b/packages/durable/test/harness-ownership.test.ts @@ -16,7 +16,6 @@ import { LiveDoc, MemoryStorage, type Storage, - StorageRejected, type Submission, type TaskId, } from "@earendil-works/pi-durable"; @@ -24,7 +23,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { chatSetup, openChat } from "./chat-support.ts"; import { addTask, addTool } from "./harness-support.ts"; -import { context } from "./session-support.ts"; +import { ControlledStorage, context } from "./session-support.ts"; import { aborted, type Deferred, deferred, openTasks, settled } from "./task-support.ts"; /** Outcome a held task commits once its gate opens. */ @@ -519,7 +518,7 @@ describe("ownership", () => { await harness.close(context); }); - it("retries marks found through an edge loaded after reopen when their commit is rejected", async () => { + it("applies marks found through an edge loaded after reopen once a failed cascade commit is reopened", async () => { const directory = await mkdtemp(join(tmpdir(), "pi-durable-ownership-")); directories.add(directory); const path = join(directory, "session.sqlite"); @@ -538,7 +537,7 @@ describe("ownership", () => { writes.some((write) => write.type === "task" && write.value.id === rejectMark && write.value.abortRequested) ) { rejectMark = undefined; - throw new StorageRejected("rejected once"); + throw new Error("disk gone"); } return commit(writes, ctx); }; @@ -549,42 +548,46 @@ describe("ownership", () => { rejectMark = id; return id; }, context); - await waitUntil(async () => rejectMark === undefined); - // Any later commit, here the reservation of the new task, retries the cascade. - expect((await opened.harness.waitForTask(late, context)).state.outcome.status).toBe("aborted"); + // The failed cascade commit fails the Harness. `closed` settles once its invocations have ended, and the owner's + // abort handler ignores its signal: let it return. open("abort.owner", "completed"); + expect(await opened.harness.closed).toMatchObject({ reason: "failed" }); + await opened.harness.close(context); + // Reopening derives the mark again. + opened = await openHarness(await openNodeSqliteStorage(path)); + expect((await opened.harness.waitForTask(late, context)).state.outcome.status).toBe("aborted"); await opened.harness.close(context); }); - it("retries a cascade whose commit the Storage rejected", async () => { - let rejectMark: TaskId | undefined; - class Rejecting extends MemoryStorage { + it("fails the Harness on a failed cascade commit, and reopening applies the cascade", async () => { + const failure = new Error("disk gone"); + let failMark: TaskId | undefined; + class Failing extends ControlledStorage { override async commit( writes: Parameters[0], ctx: Parameters[1], ) { const marks = writes.some( - (write) => write.type === "task" && write.value.id === rejectMark && write.value.abortRequested, + (write) => write.type === "task" && write.value.id === failMark && write.value.abortRequested, ); if (marks) { - rejectMark = undefined; - throw new StorageRejected("rejected once"); + failMark = undefined; + throw failure; } return super.commit(writes, ctx); } } - const { harness, root, reports } = await openHarness(new Rejecting()); + const storage = new Failing(); + const { harness, root, reports } = await openHarness(storage); const tree = await ownedChild(root, "owner"); - rejectMark = tree.inner; + failMark = tree.inner; open("owner", "failed"); - await waitUntil(async () => reports.some((error) => error instanceof StorageRejected)); - expect((await status(harness, tree.owner)).status).toBe("completing"); - expect((await status(harness, tree.inner)).status).not.toBe("terminal"); - // The next commit retries the cascade. - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); - expect((await harness.waitForTask(tree.inner, context)).state.outcome.status).toBe("aborted"); - expect((await harness.waitForTask(tree.owner, context)).state.outcome.status).toBe("failed"); + expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(reports).toEqual([failure]); await harness.close(context); + const reopened = await openHarness(storage.reopen()); + expect((await reopened.harness.waitForTask(tree.inner, context)).state.outcome.status).toBe("aborted"); + await reopened.harness.close(context); }); }); @@ -613,7 +616,7 @@ describe("owned conversations from tools and supervisors", () => { callContext, ); const settled = await submission.wait(callContext); - return { content: [{ type: "text", text: settled.status }] }; + return { output: [{ type: "text", text: settled.status }] }; }, }), ); @@ -719,7 +722,7 @@ describe("owned conversations from tools and supervisors", () => { callContext, ); try { - return { content: [{ type: "text", text: (await submission.wait(callContext)).status }] }; + return { output: [{ type: "text", text: (await submission.wait(callContext)).status }] }; } catch (error) { toolWait.resolve((error as Error).message); throw error; @@ -768,7 +771,7 @@ describe("owned conversations from tools and supervisors", () => { await childRun.reached; await handle.abort(callContext); const settledChild = await submission.wait(callContext); - return { content: [{ type: "text", text: `child ${settledChild.status}` }] }; + return { output: [{ type: "text", text: `child ${settledChild.status}` }] }; }, }), ); @@ -867,7 +870,7 @@ describe("owned conversations from tools and supervisors", () => { const request = { type: "input", content: "child task", requestId: `subagent:${api.taskId}` } as const; const submission = await (await api.conversation(child, callContext))!.submit(request, callContext); const settled = await submission.wait(callContext); - return { content: [{ type: "text", text: settled.status }] }; + return { output: [{ type: "text", text: settled.status }] }; }, }), ); diff --git a/packages/durable/test/harness-prompt.test.ts b/packages/durable/test/harness-prompt.test.ts index 8c47b19db..0ef13cc4c 100644 --- a/packages/durable/test/harness-prompt.test.ts +++ b/packages/durable/test/harness-prompt.test.ts @@ -62,7 +62,7 @@ function section(key: string, render: PromptSection["render"], const input: PromptInput = { conversationId: 1 as never, - agent: { thinkingLevel: "off", extensions: [], tools: [], sections: [] }, + agent: { thinkingLevel: "off", extensions: [], tools: [], callable: [], sections: [] }, env: undefined, shown: {}, read: { snapshot: async () => undefined, snapshotAsOf: async () => undefined }, diff --git a/packages/durable/test/harness-registry.test.ts b/packages/durable/test/harness-registry.test.ts index 3def50429..15f911484 100644 --- a/packages/durable/test/harness-registry.test.ts +++ b/packages/durable/test/harness-registry.test.ts @@ -26,7 +26,7 @@ function tool(name: string, extra: Partial = {}): AppTool { name, description: `${name} tool`, parameters: Type.Object({}), - execute: async () => ({ content: [] }), + execute: async () => ({ output: [] }), ...extra, }; } diff --git a/packages/durable/test/harness-storage-failure.test.ts b/packages/durable/test/harness-storage-failure.test.ts new file mode 100644 index 000000000..f488fc188 --- /dev/null +++ b/packages/durable/test/harness-storage-failure.test.ts @@ -0,0 +1,649 @@ +import { runInNewContext } from "node:vm"; +import type { Context } from "@earendil-works/chord"; +import { createModels } from "@earendil-works/pi-ai"; +import { + createRegistry, + defineExtension, + defineTask, + Harness, + type Id, + LiveDoc, + MemoryStorage, + StorageRequestError, + type TaskId, + watchEvents, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { safeNow } from "../src/harness/harness.ts"; +import { conversationViews } from "../src/harness/view.ts"; +import { contained } from "../src/session/session.ts"; +import { ControlledStorage, context } from "./session-support.ts"; +import { aborted, deferred, openTasks } from "./task-support.ts"; + +/** Waits for its signal, records that it saw it, and ends `aborted` through its abort handler. */ +const signals: string[] = []; +const Waiting = defineTask({ + name: "test.waiting", + version: 1, + initial: () => ({ phase: "work" }), + phases: { + work: async (_task, runtime) => { + await aborted(runtime.signal).catch(() => {}); + signals.push("aborted"); + }, + }, + abort: async (_task, runtime, abortContext) => { + await runtime.commit(() => ({ status: "terminal", outcome: { status: "aborted" } }), abortContext); + }, +}); + +/** + * Fails each Storage method named in `failing` once, with `error`. A scan of entries with a limit above 1 is the context + * read's scan of its range, off the Session line; `heldRange` holds it until released. + */ +class FailingCalls extends ControlledStorage { + readonly error = new Error("disk gone"); + readonly failing = new Set(); + heldRange: { readonly entered: ReturnType>; readonly release: Promise } | undefined; + #fail(method: string): void { + if (this.failing.delete(method)) throw this.error; + } + override async task(...args: Parameters) { + this.#fail("task"); + return super.task(...args); + } + override async scanEntries(...args: Parameters) { + if (args[1] > 1) { + const held = this.heldRange; + if (held !== undefined) { + held.entered.resolve(); + await held.release; + } + this.#fail("scanRange"); + } + return super.scanEntries(...args); + } + override async mintId>(): Promise { + this.#fail("mintId"); + return super.mintId(); + } +} + +async function started(storage: MemoryStorage) { + signals.length = 0; + const opened = await openTasks(storage, [Waiting]); + const root = await opened.harness.root(context); + return { ...opened, root }; +} + +async function running(harness: Harness, id: TaskId): Promise { + while ((await harness.getTask(id, context))?.state.status !== "running") { + await new Promise((resolve) => setTimeout(resolve, 1)); + } +} + +describe("a failed Storage call", () => { + it("ends everything with the first error: the call, later calls, waits, watches, streams, running tasks", async () => { + const storage = new ControlledStorage(); + const { harness, root, reports } = await started(storage); + const id = await root.commit( + (tx) => tx.createTask(Waiting, null, { ownership: { kind: "conversation" } }), + context, + ); + harness.resume(); + await running(harness, id); + const task = harness.waitForTask(id, context); + const idle = harness.waitForIdle(context); + const watch = (await harness.watchDoc(LiveDoc, root.id, context))!; + const stream = await watchEvents(harness, root.id, context); + stream.start(async () => {}); + + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + // The call that hit it gets the storage error itself. + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(failure); + + const failed = { name: "SessionFailed", cause: failure }; + await expect(task).rejects.toMatchObject(failed); + await expect(idle).rejects.toMatchObject(failed); + expect(await watch.closed).toEqual({ reason: "session_failed", error: failure }); + expect(await stream.closed).toEqual({ reason: "session_failed", error: failure }); + // Every later call, and a wait that starts now. + await expect(harness.getTask(id, context)).rejects.toMatchObject(failed); + await expect(harness.waitForIdle(context)).rejects.toMatchObject(failed); + await expect(root.submit({ type: "input", content: "x" }, context)).rejects.toMatchObject(failed); + expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(signals).toEqual(["aborted"]); + expect(reports).toEqual([failure]); + await harness.close(context); + }); + + it("fails it from a read off the Session line, as a context read makes", async () => { + const storage = new FailingCalls(); + const { harness, root, reports } = await started(storage); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + storage.failing.add("scanRange"); + await expect(root.context(context)).rejects.toBe(storage.error); + expect(await harness.closed).toEqual({ reason: "failed", error: storage.error }); + expect(reports).toEqual([storage.error]); + await harness.close(context); + }); + + it("ends Storage calls underway with SessionFailed, and closes once they have settled", async () => { + const storage = new FailingCalls(); + const { harness, root } = await started(storage); + const release = deferred(); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + storage.heldRange = { entered: deferred(), release: release.promise }; + const reading = root.context(context); + await storage.heldRange.entered.promise; + // Another call fails the Session while the context read is underway. + storage.failing.add("mintId"); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(storage.error); + let closed = false; + void harness.closed.then(() => { + closed = true; + }); + for (let i = 0; i < 5; i++) await new Promise((resolve) => setTimeout(resolve, 0)); + expect(closed).toBe(false); + // The read succeeds at the backend, yet its caller gets the failure: nothing it read is used. + release.resolve(); + await expect(reading).rejects.toMatchObject({ name: "SessionFailed", cause: storage.error }); + expect(await harness.closed).toEqual({ reason: "failed", error: storage.error }); + await harness.close(context); + }); + + it("closes the backend only after a paged read underway, which cannot start another page", async () => { + const pages: string[] = []; + const firstPage = deferred(); + const releaseFirst = deferred(); + class Paged extends ControlledStorage { + override async scanEntries(...args: Parameters) { + if (args[1] > 1) { + pages.push(args[2] === undefined ? "first" : "next"); + if (args[2] === undefined) { + firstPage.resolve(); + await releaseFirst.promise; + } + } + return super.scanEntries(...args); + } + override async close(closeContext: Context): Promise { + pages.push("close"); + return super.close(closeContext); + } + } + const storage = new Paged(); + const { harness, root } = await started(storage); + // More entries than one page of the context read. + await root.commit(async (tx) => { + for (let n = 0; n < 300; n++) await tx.appendEntry(root.id, { kind: "note" }); + }, context); + const reading = root.context(context); + await firstPage.promise; + const closing = harness.close(context); + releaseFirst.resolve(); + await expect(reading).rejects.toThrow("is closed"); + await closing; + expect(pages).toEqual(["first", "close"]); + expect(await harness.closed).toEqual({ reason: "closed" }); + }); + + it("fails it when Storage cannot close; an earlier failure stays the cause", async () => { + const closeError = new Error("close failed"); + class FailingClose extends ControlledStorage { + override async close(closeContext: Context): Promise { + await super.close(closeContext); + throw closeError; + } + } + const healthy = await started(new FailingClose()); + await expect(healthy.harness.close(context)).rejects.toBe(closeError); + expect(await healthy.harness.closed).toEqual({ reason: "failed", error: closeError }); + expect(healthy.reports).toEqual([closeError]); + + const storage = new FailingClose(); + const failed = await started(storage); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(failed.root.commit((tx) => tx.appendEntry(failed.root.id, { kind: "note" }), context)).rejects.toBe( + failure, + ); + expect(await failed.harness.closed).toEqual({ reason: "failed", error: failure }); + expect(failed.reports).toEqual([failure]); + }); + + it("rejects the Harness's own calls with SessionFailed after a failure", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await started(storage); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(failure); + const failed = { name: "SessionFailed", cause: failure }; + expect(() => harness.resume()).toThrow(expect.objectContaining(failed)); + await expect(harness.root(context)).rejects.toMatchObject(failed); + await expect(harness.conversation(root.id, context)).rejects.toMatchObject(failed); + await expect(harness.createConversation({ ownership: { kind: "ownerless" } }, context)).rejects.toMatchObject( + failed, + ); + await harness.close(context); + }); + + it("ends what a task holds with session_failed, not as cancelled by its signal", async () => { + const ends: unknown[] = []; + const watching = deferred(); + const Watcher = defineTask({ + name: "test.watcher", + version: 1, + initial: () => ({ phase: "work" }), + phases: { + work: async (_task, runtime, phaseContext) => { + const doc = (await runtime.watchDoc(LiveDoc, runtime.conversationId, phaseContext))!; + const handle = (await runtime.conversation(runtime.conversationId, phaseContext))!; + const idle = handle.waitForIdle(phaseContext).then( + () => "resolved", + (error: Error) => error.name, + ); + watching.resolve(); + ends.push(await doc.closed, await idle); + }, + }, + abort: async () => {}, + }); + const storage = new ControlledStorage(); + const opened = await openTasks(storage, [Watcher]); + const root = await opened.harness.root(context); + await root.commit((tx) => tx.createTask(Watcher, null, { ownership: { kind: "conversation" } }), context); + opened.harness.resume(); + await watching.promise; + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(failure); + await opened.harness.closed; + expect(ends).toEqual([{ reason: "session_failed", error: failure }, "SessionFailed"]); + await opened.harness.close(context); + }); + + it("fails it from a failed ID mint", async () => { + const storage = new FailingCalls(); + const { harness, root } = await started(storage); + storage.failing.add("mintId"); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(storage.error); + expect(await harness.closed).toEqual({ reason: "failed", error: storage.error }); + await harness.close(context); + }); + + it("does not commit what a callback did after it caught a failed read", async () => { + const storage = new FailingCalls(); + const { harness, root } = await started(storage); + const written = storage.admittedCommits.length; + storage.failing.add("task"); + await expect( + root.commit(async (tx) => { + await tx.task(1 as TaskId).catch(() => {}); + await tx.appendEntry(root.id, { kind: "note" }); + }, context), + ).rejects.toMatchObject({ name: "SessionFailed", cause: storage.error }); + expect(storage.admittedCommits.length).toBe(written); + await harness.close(context); + }); + + it("fails it for a StorageRequestError from a commit, whose effect is unknown", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await started(storage); + const rejection = new StorageRequestError("bad batch"); + storage.failNextCommit(rejection); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(rejection); + expect(await harness.closed).toEqual({ reason: "failed", error: rejection }); + await harness.close(context); + }); + + it("fails only the call for an invalid request", async () => { + const { harness, root, reports } = await started(new MemoryStorage()); + const entry = await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + const requests: (() => Promise)[] = [ + () => root.entries({}, 10, { after: "x" }, context), + async () => { + const page = await root.entries({ order: "ascending" }, 1, undefined, context); + return root.entries({ order: "descending" }, 1, page.next ?? { after: 1, order: "ascending" }, context); + }, + () => root.commit((tx) => tx.scanEntries({ conversationId: 999_999 as never }, 10), context), + // `pi.live` keeps no history. + () => harness.snapshotAsOf(LiveDoc as never, root.id, entry.id, context), + ]; + for (const request of requests) { + await expect(request()).rejects.toMatchObject({ name: "StorageRequestError" }); + } + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); + let ended = false; + void harness.closed.then(() => { + ended = true; + }); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect(ended).toBe(false); + expect(reports).toEqual([]); + await harness.close(context); + }); + + it("fails it for a throw in its own commit listeners, which would leave memory behind storage", async () => { + const storage = new ControlledStorage(); + const { harness, root, reports } = await started(storage); + const bug = new Error("listener bug"); + (harness as unknown as { observeCommits(listener: () => void): void }).observeCommits(() => { + throw bug; + }); + // The commit itself is durable and resolves; the Session fails behind it. + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); + expect(await harness.closed).toEqual({ reason: "failed", error: bug }); + expect(reports).toEqual([bug]); + await harness.close(context); + }); + + it("fails it for a storage error during a host close, before the backend closes", async () => { + const storage = new ControlledStorage(); + const { harness, root, reports } = await started(storage); + const held = storage.holdCommits(); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + const committing = root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + await held.entered; + const closing = harness.close(context); + held.release(); + await expect(committing).rejects.toBe(failure); + await closing; + expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(reports).toEqual([failure]); + }); + + it("ends an abort that joins a run ignoring its signal, and a reentrant report finds the Session failed", async () => { + const release = deferred(); + const entered = deferred(); + const Stubborn = defineTask({ + name: "test.stubborn", + version: 1, + initial: () => ({ phase: "work" }), + phases: { + work: async () => { + entered.resolve(); + await release.promise; + }, + }, + abort: async () => {}, + }); + const storage = new ControlledStorage(); + const registry = createRegistry(); + registry.install(defineExtension({ name: "tasks", tasks: [Stubborn] })); + let reentered: Promise | undefined; + const harness: Harness = await Harness.open( + storage, + { + models: createModels(), + registry, + onReport: () => { + reentered ??= harness.waitForIdle(context); + }, + }, + context, + ); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Stubborn, null, { ownership: { kind: "conversation" } }), + context, + ); + harness.resume(); + await entered.promise; + const held = storage.holdCommits(); + const aborting = harness.abortTask(id, context); + await held.entered; + held.release(); + // The abort mark commits and the abort joins the run, which ignores its signal; then the Session fails. + const failure = new Error("disk gone"); + await new Promise((resolve) => setTimeout(resolve, 5)); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).rejects.toBe(failure); + await expect(aborting).rejects.toMatchObject({ name: "SessionFailed", cause: failure }); + await expect(reentered).rejects.toMatchObject({ name: "SessionFailed", cause: failure }); + release.resolve(); + await harness.close(context); + }); + + it("does not fail it for a read its caller cancelled", async () => { + const controller = new AbortController(); + const cancelled = new Error("caller gave up"); + class CancelledRead extends MemoryStorage { + override async scanEntries(...args: Parameters) { + if (args[3].abortSignal === controller.signal) { + controller.abort(cancelled); + throw cancelled; + } + return super.scanEntries(...args); + } + } + const { harness, root } = await started(new CancelledRead()); + const cancelling: Context = { ...context, abortSignal: controller.signal }; + await expect(root.context(cancelling)).rejects.toBe(cancelled); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); + await harness.close(context); + }); +}); + +describe("listener and report errors", () => { + it("reports a listener or onReport whose promise rejects, without an unhandled rejection", async () => { + const unhandled: unknown[] = []; + const onUnhandled = (error: unknown) => unhandled.push(error); + process.on("unhandledRejection", onUnhandled); + try { + const { harness, root, reports } = await started(new MemoryStorage()); + const asyncThrow = async () => { + throw new Error("async listener"); + }; + harness.subscribeCommits(asyncThrow); + harness.subscribeClose(asyncThrow); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + await harness.close(context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(reports.map(String)).toEqual(["Error: async listener", "Error: async listener"]); + + const registry = createRegistry(); + const rejecting = await Harness.open( + new MemoryStorage(), + { models: createModels(), registry, onReport: asyncThrow }, + context, + ); + rejecting.subscribeCommits(() => { + throw new Error("sync listener"); + }); + const again = await rejecting.root(context); + await again.commit((tx) => tx.appendEntry(again.id, { kind: "note" }), context); + await rejecting.close(context); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(unhandled).toEqual([]); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); + + it("reports a throwing commit or close listener and runs the others; the commit stands", async () => { + const storage = new MemoryStorage(); + const { harness, root, reports } = await started(storage); + const seen: string[] = []; + harness.subscribeCommits(() => { + throw new Error("commit listener"); + }); + harness.subscribeCommits(() => seen.push("commit")); + harness.subscribeClose(() => { + throw new Error("close listener"); + }); + harness.subscribeClose(() => seen.push("close")); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); + await harness.close(context); + expect(seen).toEqual(["commit", "close"]); + expect(reports.map(String)).toEqual(["Error: commit listener", "Error: close listener"]); + }); + + it("keeps feeding the other observers of a view when one throws", async () => { + const { harness, root, reports } = await started(new MemoryStorage()); + const views = conversationViews(harness); + const failure = new Error("observer broke"); + await views.attach( + root.id, + async () => ({ + publication: () => { + throw failure; + }, + closeSession: () => {}, + }), + context, + ); + const watch = await root.watch(context); + const seen: unknown[] = []; + watch.start(async (value) => { + seen.push(value); + }); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); + while (seen.length === 0) await new Promise((resolve) => setTimeout(resolve, 1)); + expect(reports).toEqual([failure]); + await harness.close(context); + expect(await watch.closed).toEqual({ reason: "session_closed" }); + }); + + it("contains a fulfilled thenable that calls only its fulfillment handler", async () => { + const unhandled: unknown[] = []; + const onUnhandled = (error: unknown) => unhandled.push(error); + process.on("uncaughtException", onUnhandled); + process.on("unhandledRejection", onUnhandled); + try { + const errors: unknown[] = []; + // A thenable that, like some libraries' promises, calls only the handler it settles with. + // biome-ignore lint/suspicious/noThenProperty: the test is about a thenable + const thenable = { then: (resolve: () => void) => queueMicrotask(() => resolve()) }; + contained( + () => thenable, + (error) => errors.push(error), + ); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(errors).toEqual([]); + expect(unhandled).toEqual([]); + } finally { + process.off("uncaughtException", onUnhandled); + process.off("unhandledRejection", onUnhandled); + } + }); + + it("contains a rejected promise from another realm", async () => { + const unhandled: unknown[] = []; + const onUnhandled = (error: unknown) => unhandled.push(error); + process.on("unhandledRejection", onUnhandled); + try { + const foreign = runInNewContext("Promise.reject(new Error('foreign'))") as Promise; + const errors: unknown[] = []; + contained( + () => foreign, + (error) => errors.push(error), + ); + await new Promise((resolve) => setTimeout(resolve, 10)); + expect(errors.map(String)).toEqual(["Error: foreign"]); + expect(unhandled).toEqual([]); + } finally { + process.off("unhandledRejection", onUnhandled); + } + }); + + it("reports a throwing watch listener, which ends that watch", async () => { + const { harness, root, reports } = await started(new MemoryStorage()); + const watch = (await harness.watchDoc(LiveDoc, root.id, context))!; + const failure = new Error("listener broke"); + watch.start(async () => { + throw failure; + }); + await root.commit(async (tx) => { + (await tx.doc(LiveDoc, root.id)).compactions = []; + }, context); + expect(await watch.closed).toEqual({ reason: "listener_error", error: failure }); + expect(reports).toEqual([failure]); + await harness.close(context); + }); + + it("faults only the task, and ends only the watch, for a thrown value that cannot be formatted", async () => { + const unformattable = Object.create(null) as object; + const Throws = defineTask({ + name: "test.throws-unformattable", + version: 1, + initial: () => ({ phase: "work" }), + phases: { + work: async () => { + throw unformattable; + }, + }, + abort: async () => {}, + }); + const opened = await openTasks(new MemoryStorage(), [Throws]); + const root = await opened.harness.root(context); + const watch = (await opened.harness.watchDoc(LiveDoc, root.id, context))!; + watch.start(async () => { + throw unformattable; + }); + const id = await root.commit( + (tx) => tx.createTask(Throws, null, { ownership: { kind: "conversation" } }), + context, + ); + opened.harness.resume(); + expect((await opened.harness.waitForTask(id, context)).state.outcome).toEqual({ + status: "faulted", + error: { message: "[object Object]" }, + }); + await root.commit(async (tx) => { + (await tx.doc(LiveDoc, root.id)).compactions = []; + }, context); + expect(await watch.closed).toMatchObject({ reason: "listener_error" }); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context)).resolves.toBeDefined(); + await opened.harness.close(context); + }); + + it("reports a throwing clock once, also when the report handler reads the clock again", () => { + const reports: unknown[] = []; + const clock = safeNow( + () => { + throw new Error("clock broke"); + }, + (error) => { + reports.push(error); + // Reentry, as a handler that commits does: the clock is read again before this report returns. + if (reports.length < 5) clock(); + }, + ); + expect(clock()).toEqual(expect.any(Number)); + clock(); + expect(reports.map(String)).toEqual(["Error: clock broke"]); + }); + + it("keeps working when onReport throws, and falls back to Date.now when the clock throws", async () => { + const registry = createRegistry(); + registry.install(defineExtension({ name: "tasks", tasks: [Waiting] })); + const harness = await Harness.open( + new MemoryStorage(), + { + models: createModels(), + registry, + now: () => { + throw new Error("clock broke"); + }, + onReport: () => { + throw new Error("report broke"); + }, + }, + context, + ); + const root = await harness.root(context); + const id = await root.commit( + (tx) => tx.createTask(Waiting, null, { ownership: { kind: "conversation" } }), + context, + ); + harness.resume(); + await running(harness, id); + expect((await harness.getTask(id, context))?.startedAt).toEqual(expect.any(Number)); + await harness.abortTask(id, context); + expect((await harness.waitForTask(id, context)).state.outcome.status).toBe("aborted"); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-structured-output.test.ts b/packages/durable/test/harness-structured-output.test.ts new file mode 100644 index 000000000..566d21556 --- /dev/null +++ b/packages/durable/test/harness-structured-output.test.ts @@ -0,0 +1,339 @@ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { JsonValue } from "@earendil-works/chord"; +import { + type AssistantMessage, + fauxAssistantMessage, + fauxText, + fauxToolCall, + type ToolResultMessage, + Type, +} from "@earendil-works/pi-ai"; +import { + defineTool, + MemoryStorage, + NestedResultDoc, + type NestedToolExecutionResult, + type ToolExecutionResult, + type ToolRegistration, + ToolResultEntry, + ToolTask, + watchEvents, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { NodeExecutionEnv } from "../src/env/node.ts"; +import { createBashTool } from "../src/tools/index.ts"; +import { allEntries, type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; +import { addHooks, addTool } from "./harness-support.ts"; +import { context } from "./session-support.ts"; + +const Count = Type.Object({ count: Type.Number() }); + +/** A tool that returns `result`, with `extra` such as a schema. */ +function returning(name: string, result: ToolExecutionResult, extra: Partial = {}): ToolRegistration { + return defineTool({ + name, + description: name, + parameters: Type.Object({}), + execute: async () => result, + ...extra, + }); +} + +function call(name: string): AssistantMessage { + return fauxAssistantMessage([fauxToolCall(name, {}, { id: "c1" })], { stopReason: "toolUse" }); +} + +const DONE = fauxAssistantMessage([fauxText("done")]); + +/** Run `name` as a nested call of a `probe` tool and return what the probe received. */ +async function nested(setup: ChatSetup, name: string): Promise { + let received: NestedToolExecutionResult | undefined; + addTool( + setup.registry, + defineTool({ + name: "probe", + description: "probe", + parameters: Type.Object({}), + execute: async (_args, api, callContext) => { + received = await api.executeTool(name, {}, callContext); + return {}; + }, + }), + ); + setup.faux.setResponses([call("probe"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await harness.close(context); + return received!; +} + +/** Run `name` as a model-issued call and return its transcript message. */ +async function direct(setup: ChatSetup, name: string): Promise { + setup.faux.setResponses([call(name), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + const entry = (await allEntries(root)).find((candidate) => ToolResultEntry.is(candidate))!; + await harness.close(context); + return entry.model![0] as ToolResultMessage; +} + +function codes(result: { readonly diagnostics?: readonly { readonly code?: string }[] }): (string | undefined)[] { + return (result.diagnostics ?? []).map((diagnostic) => diagnostic.code); +} + +describe("structured output", () => { + it("gives callers a schema-less tool's output: one text as a string, one image as itself, else the list", async () => { + const text = { type: "text" as const, text: "one" }; + const image = { type: "image" as const, data: "AAAA", mimeType: "image/png" }; + const cases: [output: (typeof text | typeof image)[], value: JsonValue][] = [ + [[text], "one"], + [[image], image], + [[], ""], + [ + [text, image], + [text, image], + ], + [ + [text, text], + [text, text], + ], + ]; + for (const [output, value] of cases) { + const setup = chatSetup(); + addTool(setup.registry, returning("plain", { output })); + const received = await nested(setup, "plain"); + expect(received).toEqual({ + taskId: received.taskId, + structuredOutput: value, + isError: false, + diagnostics: [], + durationMs: received.durationMs, + }); + } + }); + + it("gives callers the bounded output, as the model sees it, without the rendered diagnostics", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + returning( + "long", + { output: [{ type: "text", text: "a\nb\nc" }], diagnostics: [{ severity: "info", message: "note" }] }, + { outputLimits: { maxLines: 1 } }, + ), + ); + const received = await nested(setup, "long"); + expect(received.structuredOutput).toBe("a\n"); + expect(codes(received)).toEqual([undefined, "truncated"]); + }); + + it("stores exactly what callers and events get, without the model's output, and keeps a stored null", async () => { + const setup = chatSetup(); + addTool(setup.registry, returning("plain", { output: [{ type: "text", text: "a" }] })); + addTool( + setup.registry, + returning("nothing", { structuredOutput: null }, { structuredOutputSchema: Type.Null() }), + ); + const received: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + defineTool({ + name: "probe", + description: "probe", + parameters: Type.Object({}), + execute: async (_args, api, callContext) => { + received.push(await api.executeTool("plain", {}, callContext)); + received.push(await api.executeTool("nothing", {}, callContext)); + return {}; + }, + }), + ); + setup.faux.setResponses([call("probe"), DONE]); + const storage = new MemoryStorage(); + const stored: JsonValue[] = []; + const commit = storage.commit.bind(storage); + storage.commit = (writes, commitContext) => { + for (const write of writes) { + if (write.type === "document.create" && write.record.kind === NestedResultDoc.definition.kind) { + stored.push(write.content.value.result!); + } + } + return commit(writes, commitContext); + }; + const { harness, root } = await openChat(storage, setup); + const stream = await watchEvents(harness, root.id, context); + const ends: NestedToolExecutionResult[] = []; + stream.start(async (batch) => { + for (const event of batch) { + if (event.type === "tool_execution_end" && event.parentToolCallId !== undefined) { + ends.push(event.result as NestedToolExecutionResult); + } + } + }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await harness.waitForIdle(context); + await stream.stop(); + await harness.close(context); + + expect(received[0]).toMatchObject({ structuredOutput: "a", isError: false }); + expect(received[1]).toMatchObject({ structuredOutput: null, isError: false }); + expect(received[0]).not.toHaveProperty("output"); + expect(stored).toEqual(received); + expect(ends).toEqual(received); + }); + + it("gives callers a schema tool's validated structured output, which the transcript does not store", async () => { + const setup = chatSetup(); + const result = { output: [{ type: "text" as const, text: "3 things" }], structuredOutput: { count: 3 } }; + addTool(setup.registry, returning("counter", result, { structuredOutputSchema: Count })); + expect(await nested(setup, "counter")).toMatchObject({ structuredOutput: { count: 3 }, isError: false }); + + const modelSetup = chatSetup(); + addTool(modelSetup.registry, returning("counter", result, { structuredOutputSchema: Count })); + const stored = await direct(modelSetup, "counter"); + expect(stored).toMatchObject({ isError: false, content: [{ type: "text", text: "3 things" }] }); + expect(stored).not.toHaveProperty("structuredOutput"); + }); + + it("turns structured output that breaks the schema into an error result for callers, and a report for the model's calls", async () => { + const cases: [name: string, result: ToolExecutionResult, schema: boolean, message: string][] = [ + ["wrong", { structuredOutput: { count: "three" } }, true, "does not match its schema: count:"], + ["missing", { output: [] }, true, "returned no structuredOutput"], + ["undeclared", { structuredOutput: { count: 3 } }, false, "declares no structuredOutputSchema"], + ]; + for (const [name, result, schema, message] of cases) { + const setup = chatSetup(); + addTool(setup.registry, returning(name, result, schema ? { structuredOutputSchema: Count } : {})); + const received = await nested(setup, name); + expect(received.isError).toBe(true); + // The broken value is dropped; a schema-less tool's error result still carries its output, here none. + if (schema) expect(received).not.toHaveProperty("structuredOutput"); + else expect(received.structuredOutput).toBe(""); + expect(codes(received)).toEqual(["invalid_structured_output"]); + expect(received.diagnostics[0]?.message).toContain(message); + + const modelSetup = chatSetup(); + addTool(modelSetup.registry, returning(name, result, schema ? { structuredOutputSchema: Count } : {})); + // The model never sees structured output: its call stands, and the host gets a report. + const stored = await direct(modelSetup, name); + expect(stored.isError).toBe(false); + expect(JSON.stringify(stored.content)).not.toContain(message); + expect(modelSetup.reports.map((error) => (error as Error).message).join("\n")).toContain(message); + } + }); + + it("gives callers the output of an error result of a tool without a schema", async () => { + const setup = chatSetup(); + addTool(setup.registry, returning("bad", { output: [{ type: "text", text: "bad" }], isError: true })); + expect(await nested(setup, "bad")).toMatchObject({ isError: true, structuredOutput: "bad" }); + }); + + it("gives callers no structured output for an error result the Harness wrote, only diagnostics", async () => { + const setup = chatSetup(); + const received = await nested(setup, "missing"); + expect(received).toMatchObject({ isError: true, diagnostics: [{ code: "tool_unavailable" }] }); + expect(received).not.toHaveProperty("structuredOutput"); + }); + + it("lets an error result of a schema tool omit structured output, and keeps none of its output", async () => { + const setup = chatSetup(); + addTool( + setup.registry, + returning( + "fails", + { output: [{ type: "text", text: "nope" }], isError: true }, + { structuredOutputSchema: Count }, + ), + ); + const received = await nested(setup, "fails"); + expect(received).toMatchObject({ isError: true, diagnostics: [] }); + expect(received).not.toHaveProperty("structuredOutput"); + }); + + it("keeps structured output when afterTool replaces the output, and validates one afterTool replaces", async () => { + const setup = chatSetup(); + addTool(setup.registry, returning("plain", { output: [{ type: "text", text: "secret" }] })); + addTool( + setup.registry, + returning("counter", { structuredOutput: { count: 1 } }, { structuredOutputSchema: Count }), + ); + addTool( + setup.registry, + returning("other", { structuredOutput: { count: 1 } }, { structuredOutputSchema: Count }), + ); + addHooks(setup.registry, ToolTask, { + afterTool: (hookCall, result) => + hookCall.name === "other" + ? { ...result, structuredOutput: { count: "broken" } as unknown as JsonValue } + : hookCall.name === "probe" + ? result + : { ...result, output: [{ type: "text", text: "rewritten" }] }, + }); + let plain: NestedToolExecutionResult | undefined; + let counter: NestedToolExecutionResult | undefined; + let other: NestedToolExecutionResult | undefined; + addTool( + setup.registry, + defineTool({ + name: "probe", + description: "probe", + parameters: Type.Object({}), + execute: async (_args, api, callContext) => { + plain = await api.executeTool("plain", {}, callContext); + counter = await api.executeTool("counter", {}, callContext); + other = await api.executeTool("other", {}, callContext); + return {}; + }, + }), + ); + setup.faux.setResponses([call("probe"), DONE]); + const { harness, root } = await openChat(new MemoryStorage(), setup); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await harness.close(context); + // A schema-less tool's value is its output, so redacting the output redacts what programs get. + expect(plain?.structuredOutput).toBe("rewritten"); + expect(counter).toMatchObject({ structuredOutput: { count: 1 } }); + expect(other?.isError).toBe(true); + expect(codes(other!)).toEqual(["invalid_structured_output"]); + }); + + it("gives callers of bash its retained output and exit code, also for a nonzero exit", async () => { + const directory = mkdtempSync(join(tmpdir(), "pi-durable-structured-")); + try { + const setup = chatSetup(); + addTool(setup.registry, createBashTool()); + const received: NestedToolExecutionResult[] = []; + addTool( + setup.registry, + defineTool({ + name: "probe", + description: "probe", + parameters: Type.Object({}), + execute: async (_args, api, callContext) => { + received.push(await api.executeTool("bash", { command: "printf ok" }, callContext)); + received.push(await api.executeTool("bash", { command: "printf bad; exit 4" }, callContext)); + return {}; + }, + }), + ); + setup.faux.setResponses([call("probe"), DONE]); + const env = new NodeExecutionEnv({ cwd: directory }); + const { harness, root } = await openChat(new MemoryStorage(), setup, { env }); + await (await root.submit({ type: "input", content: "go" }, context)).wait(context); + await harness.close(context); + expect(received[0]).toMatchObject({ + isError: false, + structuredOutput: { output: "ok", truncated: false, exitCode: 0 }, + }); + expect(received[1]).toMatchObject({ + isError: true, + structuredOutput: { output: "bad", truncated: false, exitCode: 4 }, + }); + expect(codes(received[1]!)).toEqual(["exit_code"]); + } finally { + rmSync(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/packages/durable/test/harness-structured.test.ts b/packages/durable/test/harness-structured.test.ts index f061e1055..d48adccca 100644 --- a/packages/durable/test/harness-structured.test.ts +++ b/packages/durable/test/harness-structured.test.ts @@ -26,7 +26,6 @@ import { type NextTaskState, type Registry, type Storage, - StorageRejected, type TaskId, type TaskOptions, type TaskRuntime, @@ -39,7 +38,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, waitFor } from "./chat-support.ts"; import { addHooks, addTask, addTool } from "./harness-support.ts"; -import { context } from "./session-support.ts"; +import { ControlledStorage, context } from "./session-support.ts"; import { aborted, type Deferred, deferred, openTasks, settled } from "./task-support.ts"; // ─── A scriptable task ────────────────────────────────────────────────────── @@ -1019,7 +1018,7 @@ describe("tool rounds", () => { name: "noop", description: "Does nothing", parameters: Type.Object({}), - execute: async () => ({ content: [] }), + execute: async () => ({ output: [] }), }), ); setup.faux.setResponses([toolCalls(["noop", "c1"], ["noop", "c2"]), fauxAssistantMessage([fauxText("done")])]); @@ -1097,7 +1096,7 @@ describe("tool rounds", () => { name: "noop", description: "Does nothing", parameters: Type.Object({}), - execute: async () => ({ content: [] }), + execute: async () => ({ output: [] }), }), ); // An extension's hook starts work owned by the generation, which holds it while the next turn runs. @@ -1128,7 +1127,7 @@ describe("tool rounds", () => { await harness.close(context); }); - it("keeps run control with a faulted generation until its owned work drains, and retries a rejected final commit", async () => { + it("keeps run control with a faulted generation until its owned work drains, and fails the Harness on a failed final commit", async () => { const base = chatSetup(); const models = new Proxy(base.models, { get(target, property) { @@ -1153,7 +1152,8 @@ describe("tool rounds", () => { }, }); let reject: TaskId | undefined; - class Rejecting extends MemoryStorage { + const failure = new Error("disk gone"); + class Failing extends ControlledStorage { override async commit( writes: Parameters[0], ctx: Parameters[1], @@ -1163,12 +1163,13 @@ describe("tool rounds", () => { ); if (finalizes) { reject = undefined; - throw new StorageRejected("rejected once"); + throw failure; } return super.commit(writes, ctx); } } - const { harness, root } = await openChat(new Rejecting(), setup); + const storage = new Failing(); + const { harness, root } = await openChat(storage, setup); opened = harness; const submission = await root.submit({ type: "input", content: "hi" }, context); await waitFor(async () => { @@ -1183,17 +1184,18 @@ describe("tool rounds", () => { await waitFor(() => log.includes("abort:hooked")); reject = generation; open("abort.hooked"); - // The final commit, with the run's cleanup, is rejected once: nothing of the cleanup lands. - await waitFor(() => setup.reports.some((error) => error instanceof StorageRejected)); - expect((await harness.snapshot(LiveDoc, root.id, context))!.run?.taskId).toBe(generation); - expect((await allEntries(root)).map((entry) => entry.kind)).toEqual(["pi.user"]); - expect(await settled(submission.wait(context))).toBe(false); - // The next commit retries it; the partial becomes one aborted entry. - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); - expect(await submission.wait(context)).toMatchObject({ status: "unanswered", reason: "faulted" }); - expect(await harness.snapshot(LiveDoc, root.id, context)).toEqual({}); - expect((await allEntries(root)).map((entry) => entry.kind)).toEqual(["pi.user", "note", "pi.assistant"]); + // The final commit, with the run's cleanup, fails: nothing of the cleanup lands, and the Harness fails. + expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + await expect(submission.wait(context)).rejects.toMatchObject({ name: "SessionFailed", cause: failure }); await harness.close(context); + // Reopening finalizes the run; the partial becomes one aborted entry. + const reopened = await openChat(storage.reopen(), setup); + opened = reopened.harness; + const again = (await reopened.harness.submission(submission.id, context))!; + expect(await again.wait(context)).toMatchObject({ status: "unanswered", reason: "faulted" }); + expect(await reopened.harness.snapshot(LiveDoc, reopened.root.id, context)).toEqual({}); + expect((await allEntries(reopened.root)).map((entry) => entry.kind)).toEqual(["pi.user", "pi.assistant"]); + await reopened.harness.close(context); }); }); @@ -1425,9 +1427,10 @@ describe("conversation abort and boundaries", () => { await harness.close(context); }); - it("retries a finalization the Storage rejected with the next commit", async () => { + it("fails the Harness on a failed finalization, which reopening applies", async () => { let reject: TaskId | undefined; - class Rejecting extends MemoryStorage { + const failure = new Error("disk gone"); + class Failing extends ControlledStorage { override async commit( writes: Parameters[0], ctx: Parameters[1], @@ -1437,23 +1440,24 @@ describe("conversation abort and boundaries", () => { ); if (finalizes) { reject = undefined; - throw new StorageRejected("rejected once"); + throw failure; } return super.commit(writes, ctx); } } - const { harness, root, reports } = await openNodes(new Rejecting()); - const child = spawnAndFinish("parent", "child"); + const storage = new Failing(); + const { harness, root, reports } = await openNodes(storage); + spawnAndFinish("parent", "child"); const parent = await start(root, "parent"); await until(async () => (await state(harness, parent)).status === "completing"); reject = parent; open("child"); - await harness.waitForTask(child.id!, context); - await until(() => reports.some((error) => error instanceof StorageRejected)); - expect((await state(harness, parent)).status).toBe("completing"); - await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); - expect(await outcomeOf(harness, parent)).toBe("completed"); + expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(reports).toEqual([failure]); await harness.close(context); + const reopened = await openNodes(storage.reopen()); + expect(await outcomeOf(reopened.harness, parent)).toBe("completed"); + await reopened.harness.close(context); }); }); @@ -1605,7 +1609,7 @@ describe("tool rounds and events", () => { name: "noop", description: "Does nothing", parameters: Type.Object({}), - execute: async () => ({ content: [] }), + execute: async () => ({ output: [] }), }), ); let opened: Harness | undefined; @@ -1647,7 +1651,7 @@ describe("tool rounds and events", () => { const child = await tx.createConversation({ ownership: { kind: "task", taskId: api.taskId } }); await tx.createTask(Node, { name: "sub" }, { ...OWN_CONVERSATION, conversationId: child.id }); }, callContext); - return { content: [{ type: "text", text: "started" }] }; + return { output: [{ type: "text", text: "started" }] }; }, }), ); @@ -1687,7 +1691,7 @@ describe("tool rounds and events", () => { parameters: Type.Object({}), execute: async (_args, api, callContext) => { await api.commit((tx) => tx.createTask(Node, { name: "held" }, owned(api.taskId)), callContext); - return { content: [], details: { fn: (() => 1) as unknown as JsonValue } }; + return { output: [], details: { fn: (() => 1) as unknown as JsonValue } }; }, }), ); diff --git a/packages/durable/test/harness-support.ts b/packages/durable/test/harness-support.ts index fc04a3b49..956b2e675 100644 --- a/packages/durable/test/harness-support.ts +++ b/packages/durable/test/harness-support.ts @@ -26,7 +26,7 @@ import { import { context } from "./session-support.ts"; export function tool(name: string, description = `${name} tool`): ToolRegistration { - return defineTool({ name, description, parameters: Type.Object({}), execute: async () => ({ content: [] }) }); + return defineTool({ name, description, parameters: Type.Object({}), execute: async () => ({ output: [] }) }); } /** Open a Harness with a fresh registry holding the named tools. */ diff --git a/packages/durable/test/harness-tasks.test.ts b/packages/durable/test/harness-tasks.test.ts index f88b25567..55aabb697 100644 --- a/packages/durable/test/harness-tasks.test.ts +++ b/packages/durable/test/harness-tasks.test.ts @@ -13,7 +13,6 @@ import { MemoryStorage, type RegistryReader, type RegistrySnapshot, - StorageRejected, type TaskId, type TaskRuntime, } from "@earendil-works/pi-durable"; @@ -743,76 +742,65 @@ describe("task runtime", () => { }); describe("task scheduling", () => { - it("retries reservation on the next wakeup after a rejected reservation commit", async () => { + it("fails the Harness on a failed reservation commit; reopening runs the task", async () => { let runs = 0; const Once = oneStep("test.once", async (_task, runtime, ctx) => { runs++; await runtime.commit(() => completed(null), ctx); }); const storage = new ControlledStorage(); - const { harness, root, registry, reports } = await openRoot([Once], { storage }); + const { harness, root, reports } = await openRoot([Once], { storage }); const id = await start(root, Once); - storage.failNextCommit(new StorageRejected("busy")); + const waiting = harness.waitForTask(id, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); harness.resume(); - await eventually(() => reports.length === 1); - expect((await harness.getTask(id, context))?.state.status).toBe("pending"); - // Any wakeup, here a registry change, reserves again. - addTool( - registry, - defineTool({ - name: "wake", - description: "wake", - parameters: Type.Object({}), - execute: async () => ({}), - }), - ); - await harness.waitForTask(id, context); + const end = await harness.closed; + expect(end).toEqual({ reason: "failed", error: failure }); + expect(reports).toEqual([failure]); + // The pending wait and every later call get SessionFailed with the storage error. + await expect(waiting).rejects.toMatchObject({ name: "SessionFailed", cause: failure }); + await expect(harness.getTask(id, context)).rejects.toMatchObject({ name: "SessionFailed", cause: failure }); + await expect(root.submit({ type: "input", content: "x" }, context)).rejects.toMatchObject({ + name: "SessionFailed", + }); + expect(runs).toBe(0); + await harness.close(context); + + const reopened = await openRoot([Once], { storage: storage.reopen() }); + reopened.harness.resume(); + expect((await reopened.harness.waitForTask(id, context)).state.outcome.status).toBe("completed"); expect(runs).toBe(1); - await harness.close(context); + await reopened.harness.close(context); }); - it("keeps a wakeup that arrives while a rejected reservation commit is in storage", async () => { - const First = oneStep("test.wake-first", async (_task, runtime, ctx) => { - await runtime.commit(() => completed(null), ctx); - }); - const Late = oneStep("test.wake-late", async (_task, runtime, ctx) => { - await runtime.commit(() => completed(null), ctx); - }); - const storage = new ControlledStorage(); - const { harness, root, registry } = await openRoot([First], { storage }); - const first = await start(root, First); - const late = await start(root, Late); - const held = storage.holdCommits(); - storage.failNextCommit(new StorageRejected("busy")); - harness.resume(); - await held.entered; - // Registering the missing definition wakes the scheduler while the doomed reservation is in storage. - addTask(registry, Late); - held.release(); - await harness.waitForTask(first, context); - await harness.waitForTask(late, context); - await harness.close(context); - }); - - it("reruns a task whose fault write was rejected", async () => { + it("fails the Harness instead of running a task again when its fault write fails", async () => { let runs = 0; const storage = new ControlledStorage(); - const Throws = oneStep("test.rejected-fault", async () => { + const failure = new Error("disk gone"); + const Throws = oneStep("test.failed-fault", async () => { runs++; // The next commit is the step's fault write. - if (runs === 1) storage.failNextCommit(new StorageRejected("busy")); + if (runs === 1) storage.failNextCommit(failure); throw new Error("boom"); }); const { harness, root, reports } = await openRoot([Throws], { storage }); const id = await start(root, Throws); harness.resume(); - expect((await harness.waitForTask(id, context)).state.outcome).toEqual({ + expect(await harness.closed).toEqual({ reason: "failed", error: failure }); + expect(runs).toBe(1); + expect(reports).toEqual([failure]); + await harness.close(context); + + // Reopening recovers the task as after a crash: still running, so its phase runs again. + const reopened = await openRoot([Throws], { storage: storage.reopen() }); + reopened.harness.resume(); + expect((await reopened.harness.waitForTask(id, context)).state.outcome).toEqual({ status: "faulted", error: { message: "boom" }, }); expect(runs).toBe(2); - expect(reports.map(String)).toEqual(["StorageRejected: busy"]); - await harness.close(context); + await reopened.harness.close(context); }); it("waits for Harness and conversation idleness, counting blocked work and ignoring background tasks", async () => { diff --git a/packages/durable/test/harness-tools-recovery.test.ts b/packages/durable/test/harness-tools-recovery.test.ts index a03932f96..226bade81 100644 --- a/packages/durable/test/harness-tools-recovery.test.ts +++ b/packages/durable/test/harness-tools-recovery.test.ts @@ -218,7 +218,7 @@ describe("tool recovery", () => { setup.registry, tool("work", async () => { runs++; - return { content: [] }; + return { output: [] }; }), ); const reached = deferred(); @@ -254,7 +254,7 @@ describe("tool recovery", () => { const setup = chatSetup(); addTool( setup.registry, - tool("work", async () => ({ content: [] })), + tool("work", async () => ({ output: [] })), ); const reached = deferred(); let observed = 0; @@ -313,7 +313,7 @@ describe("tool recovery", () => { // A result that is not strict JSON makes the result commit throw, so the scheduler faults the task. addTool( setup.registry, - tool("bad", async () => ({ content: [], details: { fn: (() => 1) as unknown as JsonValue } })), + tool("bad", async () => ({ output: [], details: { fn: (() => 1) as unknown as JsonValue } })), ); const requests: string[] = []; setup.faux.setResponses([ diff --git a/packages/durable/test/harness-tools.test.ts b/packages/durable/test/harness-tools.test.ts index 83ff0e015..75b8a9bd7 100644 --- a/packages/durable/test/harness-tools.test.ts +++ b/packages/durable/test/harness-tools.test.ts @@ -84,7 +84,7 @@ describe("tool round", () => { addTool( setup.registry, tool("echo", async (args) => ({ - content: [{ type: "text", text: `echo ${args.text}` }], + output: [{ type: "text", text: `echo ${args.text}` }], })), ); const { harness, root, entries, status } = await run(setup, [calls(["echo", { text: "hi" }, "c1"]), DONE]); @@ -119,7 +119,7 @@ describe("tool round", () => { setup.registry, tool("echo", async (_args, api) => { seen.push(api.models); - return { content: [] }; + return { output: [] }; }), ); addHooks(setup.registry, ToolTask, { @@ -139,7 +139,7 @@ describe("tool round", () => { const setup = chatSetup(); addTool( setup.registry, - tool("echo", async () => ({ content: [] })), + tool("echo", async () => ({ output: [] })), ); const { harness, root, entries, status } = await run(setup, [ calls(["ghost", {}, "c1"], ["echo", {}, "c2"]), @@ -166,7 +166,7 @@ describe("tool round", () => { setup.registry, tool("echo", async () => { seen.push("ran"); - return { content: [] }; + return { output: [] }; }), ); let root!: Conversation; @@ -187,7 +187,7 @@ describe("tool round", () => { it("removes unregistered active tools from the offer and adds them back after re-registration", async () => { const setup = chatSetup(); - const echo = tool("echo", async () => ({ content: [] })); + const echo = tool("echo", async () => ({ output: [] })); const registration = addTool(setup.registry, echo); const first = await run(setup, [DONE]); registration.dispose(); @@ -219,14 +219,14 @@ describe("tool round", () => { "first", async () => { second.dispose(); - return { content: [] }; + return { output: [] }; }, { executionMode: "sequential" }, ), ); second = addTool( setup.registry, - tool("second", async () => ({ content: [{ type: "text", text: "ran" }] })), + tool("second", async () => ({ output: [{ type: "text", text: "ran" }] })), ); const { harness, entries, status } = await run(setup, [calls(["first", {}, "c1"], ["second", {}, "c2"]), DONE]); expect(status).toBe("done"); @@ -246,7 +246,7 @@ describe("tool round", () => { setup.settings.toolExecution = "parallel"; await new Promise((resolve) => setTimeout(resolve, 20)); events.push(`end ${name}`); - return { content: [] }; + return { output: [] }; }; addTool(setup.registry, tool("a", slow("a"))); addTool(setup.registry, tool("b", slow("b"))); @@ -269,7 +269,7 @@ describe("tool round", () => { events.push(`start ${name}`); await new Promise((resolve) => setTimeout(resolve, 20)); events.push(`end ${name}`); - return { content: [] }; + return { output: [] }; }; if (setup.registry.snapshot().extension("tool:a") === undefined) addTool(setup.registry, tool("a", slow("a"))); if (setup.registry.snapshot().extension("tool:b") === undefined) addTool(setup.registry, tool("b", slow("b"))); @@ -293,7 +293,7 @@ describe("tool round", () => { async () => { await new Promise((resolve) => setTimeout(resolve, 20)); events.push("a"); - return { content: [] }; + return { output: [] }; }, { executionMode: "sequential" }, ), @@ -302,7 +302,7 @@ describe("tool round", () => { perTool.registry, tool("b", async () => { events.push("b"); - return { content: [] }; + return { output: [] }; }), ); const result = await run(perTool, [calls(["a", {}, "c1"], ["b", {}, "c2"]), DONE]); @@ -399,7 +399,7 @@ describe("tool results", () => { tool( "big", async () => ({ - content: [{ type: "text", text: "a\nb\n" }, image, { type: "text", text: "c\nd\n" }], + output: [{ type: "text", text: "a\nb\n" }, image, { type: "text", text: "c\nd\n" }], }), { outputLimits: { maxLines: 2, retain: "tail" } }, ), @@ -436,7 +436,7 @@ describe("tool results", () => { setup.registry, tool("slow", async () => { await sleep(30); - return { content: [] }; + return { output: [] }; }), ); addTool( @@ -473,7 +473,7 @@ describe("tool results", () => { setup.registry, tool("echo", async (args) => { seen.push(args); - return { content: [] }; + return { output: [] }; }), ); addHooks(setup.registry, ToolTask, { @@ -520,7 +520,7 @@ describe("tool results", () => { "echo", async (args) => { seen.push(args); - return { content: [] }; + return { output: [] }; }, { prepareArguments: (args) => { @@ -549,7 +549,7 @@ describe("tool results", () => { const setup = chatSetup(); addTool( setup.registry, - tool("echo", async () => ({ content: [] })), + tool("echo", async () => ({ output: [] })), ); const asked: string[] = []; addHooks(setup.registry, ToolTask, { @@ -574,13 +574,18 @@ describe("tool results", () => { const setup = chatSetup(); addTool( setup.registry, - tool("echo", async () => ({ content: [{ type: "text", text: "raw" }] })), + tool("echo", async () => ({ output: [{ type: "text", text: "raw" }] })), ); addHooks(setup.registry, ToolTask, { - afterTool: (_call, result) => ({ ...result, content: [{ type: "text", text: "first" }] }), + afterTool: (_call, result) => ({ ...result, output: [{ type: "text", text: "first" }] }), }); addHooks(setup.registry, ToolTask, { - afterTool: (_call, result) => ({ ...result, details: { replaced: resultText(result as ToolResultMessage) } }), + afterTool: (_call, result) => ({ + ...result, + details: { + replaced: (result.output ?? []).map((item) => (item.type === "text" ? item.text : "")).join("|"), + }, + }), }); const observed: unknown[] = []; addHooks(setup.registry, GenerationTask, { @@ -598,7 +603,7 @@ describe("tool results", () => { it("runs the hooks of the selected extensions; a task-owned child copies its owner's selection", async () => { const setup = chatSetup(); const calledIn: number[] = []; - const Echo = defineExtension({ name: "echo", tools: [tool("echo", async () => ({ content: [] }))] }); + const Echo = defineExtension({ name: "echo", tools: [tool("echo", async () => ({ output: [] }))] }); const Audit = defineExtension({ name: "audit", hooks: [hook(ToolTask, { beforeTool: (_call, api) => void calledIn.push(api.conversationId) })], @@ -640,13 +645,13 @@ describe("tool results", () => { it("applies addTools and terminates only when every result of the round asks to", async () => { const setup = chatSetup(); - const stop = tool("stop", async () => ({ content: [], control: { terminate: true } })); - const grow = tool("grow", async () => ({ content: [], control: { addTools: ["extra", "stop"] } })); + const stop = tool("stop", async () => ({ output: [], control: { terminate: true } })); + const grow = tool("grow", async () => ({ output: [], control: { addTools: ["extra", "stop"] } })); addTool(setup.registry, stop); addTool(setup.registry, grow); addTool( setup.registry, - tool("extra", async () => ({ content: [] })), + tool("extra", async () => ({ output: [] })), ); const first = await run(setup, [calls(["stop", {}, "c1"])], async (_harness, root) => { await root.configure({ tools: [stop, grow] }, context); @@ -780,7 +785,7 @@ describe("generation hooks", () => { let asked = 0; addTool( setup.registry, - tool("echo", async () => ({ content: [] })), + tool("echo", async () => ({ output: [] })), ); addHooks(setup.registry, ToolTask, { beforeTool: async (_call, api) => { @@ -829,7 +834,7 @@ describe("tool execution api", () => { const id = await api.createTask(child, { n: 21 }, { ownership: { kind: "conversation" } }, context); const done = await api.waitForTask(id, context); seen.push(done.state.outcome); - return { content: [] }; + return { output: [] }; }); addTool(setup.registry, probe); setup.faux.setResponses([calls(["probe", {}, "c1"], ["probe", {}, "c2"]), DONE]); @@ -855,7 +860,7 @@ describe("tool execution api", () => { const setup = chatSetup(); addTool( setup.registry, - tool("probe", async () => ({ content: [{ type: "text", text: "ran" }] })), + tool("probe", async () => ({ output: [{ type: "text", text: "ran" }] })), ); const { harness, entries } = await run(setup, [calls(["probe", {}, "c1"]), DONE], undefined, { env: () => { @@ -889,7 +894,7 @@ describe("coding tools", () => { expect(text.startsWith("line-1001\n")).toBe(true); expect(ToolResultEntry.is(entry) && entry.data.diagnostics.map((diagnostic) => diagnostic.code)).toEqual([ "full_output", - "tool_error", + "exit_code", "truncated", ]); expect(text).toContain("line-3000\n|\n[info] Full output: "); @@ -970,7 +975,7 @@ describe("tool progress and lifetime", () => { ); addTool( setup.registry, - tool("explicit", async () => ({ content: [{ type: "text", text: "c\u001bd" }] })), + tool("explicit", async () => ({ output: [{ type: "text", text: "c\u001bd" }] })), ); const { harness, entries } = await run(setup, [calls(["noisy", {}, "c1"], ["explicit", {}, "c2"]), DONE]); expect(slotOutput).toBe("ab\n"); @@ -984,9 +989,9 @@ describe("tool progress and lifetime", () => { const setup = chatSetup(); addTool( setup.registry, - tool("grow", async () => ({ content: [], control: { addTools: ["extra"], terminate: undefined } })), + tool("grow", async () => ({ output: [], control: { addTools: ["extra"], terminate: undefined } })), ); - const extra = tool("extra", async () => ({ content: [] })); + const extra = tool("extra", async () => ({ output: [] })); addTool(setup.registry, extra); const { harness, root, status } = await run(setup, [calls(["grow", {}, "c1"]), DONE], (_harness, conversation) => conversation.configure({ tools: { remove: [extra] } }, context), @@ -1003,7 +1008,7 @@ describe("tool progress and lifetime", () => { setup.registry, tool("null", async (_args, api) => { await api.details({ old: 1 }, context); - return { content: [], details: null }; + return { output: [], details: null }; }), ); const { harness, entries } = await run(setup, [calls(["null", {}, "c1"]), DONE]); @@ -1023,7 +1028,7 @@ describe("tool progress and lifetime", () => { await first; void second; void api.details({ n: 3 }, context).then(() => settled.push("third")); - return { content: [] }; + return { output: [] }; }), ); const { harness, entries } = await run(setup, [calls(["details", {}, "c1"]), DONE]); @@ -1045,7 +1050,7 @@ describe("tool progress and lifetime", () => { const v1 = tool("work", async () => { release(); await finished; - return { content: [{ type: "text", text: "v1" }] }; + return { output: [{ type: "text", text: "v1" }] }; }); addTool(setup.registry, v1); setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); @@ -1055,7 +1060,7 @@ describe("tool progress and lifetime", () => { // The same extension name replaces the old one in place. addTool( setup.registry, - tool("work", async () => ({ content: [{ type: "text", text: "v2" }] })), + tool("work", async () => ({ output: [{ type: "text", text: "v2" }] })), ); finish(); await submission.wait(context); @@ -1080,7 +1085,7 @@ describe("tool progress and lifetime", () => { tool("work", async () => { running.resolve(); await reloaded.promise; - return { content: [] }; + return { output: [] }; }), ); setup.faux.setResponses([calls(["work", {}, "c1"]), DONE]); @@ -1119,7 +1124,7 @@ describe("tool progress and lifetime", () => { wait.catch(() => {}); const watch = await api.watchDoc(LiveDoc, api.conversationId, context); watchClosed = watch!.closed; - return { content: [] }; + return { output: [] }; }), ); const { harness } = await run(setup, [calls(["detach", {}, "c1"]), DONE]); diff --git a/packages/durable/test/nested-images-bench.ts b/packages/durable/test/nested-images-bench.ts new file mode 100644 index 000000000..4030064eb --- /dev/null +++ b/packages/durable/test/nested-images-bench.ts @@ -0,0 +1,305 @@ +/** + * Nested image storage benchmark. One model-issued `script` call, as a code mode call would, loops `count` times: a + * nested `read_image` call returns a unique base64 image of `bytes` characters, then a nested `classify` call reads it. + * The `inline` variant passes the image data in `classify`'s arguments; the `reference` variant passes the `taskId` of + * the `read_image` result, whose value is the image, and `classify` resolves it from its caller's `NestedResultDoc`. + * Reports commits, wall time, logical bytes written (JSON of the storage writes) by write type, the stored size before + * `script` returns, after the run settles, after close, and after reopening, the heap (peak during the run, sampled + * every 5 ms and at each commit; after a GC once the run settles, after `waitForIdle()`, and after reopening; all over + * the heap after a GC before opening), the number of nested task records, and the size of the largest one. + * + * node --conditions=source --experimental-strip-types --expose-gc test/nested-images-bench.ts [count] + */ +import { randomBytes } from "node:crypto"; +import { mkdtemp, readdir, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; +import type { ImageContent } from "@earendil-works/pi-ai"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { Type } from "typebox"; +import { + createRegistry, + defineExtension, + defineTool, + Harness, + type JsonObject, + NestedResultDoc, + type Storage, + type StorageWrite, + ToolResultEntry, + type ToolTaskInput, +} from "../src/index.ts"; +import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; + +type Backend = "sqlite" | "jsonl"; +type Variant = "inline" | "reference"; +type Scenario = { backend: Backend; variant: Variant; count: number; bytes: number }; +/** Hooks the tools call: a heap sample, and the stored size measured before `script` returns. */ +type Probe = { sample: () => void; beforeReturn: () => Promise }; +type Written = Record; +type Result = { + commits: number; + ms: number; + written: Written; + /** Before `script` returns, while its nested results are live. */ + storedLive: number; + storedRun: number; + storedClosed: number; + storedReopened: number; + /** Over the heap after a GC before opening. */ + peakHeap: number; + settledHeap: number; + idleHeap: number; + reopenedHeap: number; + nestedTasks: number; + largestTask: number; +}; + +const gc = (globalThis as { gc?: () => void }).gc; +if (gc === undefined) throw new Error("Run with --expose-gc"); + +function heapAfterGc(): number { + gc!(); + return process.memoryUsage().heapUsed; +} + +async function directorySize(path: string): Promise { + let total = 0; + for (const entry of await readdir(path, { withFileTypes: true })) { + const full = join(path, entry.name); + total += entry.isDirectory() ? await directorySize(full) : (await stat(full)).size; + } + return total; +} + +async function open(backend: Backend, directory: string): Promise { + return backend === "sqlite" + ? openNodeSqliteStorage(join(directory, "session.sqlite")) + : openNodeJsonlStorage(directory, context); +} + +function noWrites(): Written { + return { + conversation: 0, + entry: 0, + task: 0, + submission: 0, + "document.create": 0, + "document.copy": 0, + "document.change": 0, + "document.retire": 0, + }; +} + +const ImageArgument = Type.Object({ + type: Type.Literal("image"), + mimeType: Type.String(), + data: Type.Optional(Type.String()), + source: Type.Optional(Type.Object({ taskId: Type.Number() })), +}); + +function registryFor(scenario: Scenario, probe: Probe) { + const registry = createRegistry(); + const readImage = defineTool({ + name: "read_image", + description: "Read an image", + parameters: Type.Object({ n: Type.Number() }), + // One image, so programs get it as an `ImageContent`. Unique per call: base64 of random bytes. + execute: async () => ({ + output: [ + { type: "image", data: randomBytes((scenario.bytes / 4) * 3).toString("base64"), mimeType: "image/png" }, + ], + }), + }); + const classify = defineTool({ + name: "classify", + description: "Classify images", + parameters: Type.Object({ images: Type.Array(ImageArgument) }), + execute: async (args, api, callContext) => { + for (const image of args.images) { + let data = image.data; + if (image.source !== undefined) { + // The caller is the nested call's `parent`; its `NestedResultDoc` holds the referenced result. + const input = (await api.getTask(api.taskId, callContext))?.input as ToolTaskInput | undefined; + if (input?.kind !== "nested") throw new Error("classify resolves references only as a nested call"); + const stored = await api.snapshot( + NestedResultDoc, + input.parent, + String(image.source.taskId), + callContext, + ); + const block = stored?.result.structuredOutput as ImageContent | undefined; + data = block?.type === "image" ? block.data : undefined; + } + if (data?.length !== scenario.bytes) throw new Error(`Image has ${data?.length} characters`); + } + return { output: [{ type: "text", text: "ok" }] }; + }, + }); + const script = defineTool({ + name: "script", + description: "Read and classify images", + parameters: Type.Object({ count: Type.Number() }), + callers: ["model"], + execute: async (args, api, callContext) => { + for (let n = 0; n < args.count; n++) { + const key = `read${n}`; + const read = await api.executeTool("read_image", { n }, callContext, { key }); + const image = read.structuredOutput as unknown as ImageContent; + const argument: JsonObject = + scenario.variant === "inline" + ? { type: "image", data: image.data, mimeType: image.mimeType } + : { type: "image", source: { taskId: read.taskId }, mimeType: image.mimeType }; + const classified = await api.executeTool("classify", { images: [argument] }, callContext, { + key: `classify${n}`, + }); + if (classified.isError) throw new Error(`classify ${n} failed: ${JSON.stringify(classified.diagnostics)}`); + probe.sample(); + } + await probe.beforeReturn(); + return { output: [{ type: "text", text: `classified ${args.count}` }] }; + }, + }); + registry.install(defineExtension({ name: "bench", tools: [readImage, classify, script] })); + return registry; +} + +async function runScenario(scenario: Scenario): Promise { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-images-bench-")); + try { + const faux = fauxProvider(); + const models = createModels(); + models.setProvider(faux.provider); + let peakHeap = 0; + let storedLive = 0; + const sample = () => { + peakHeap = Math.max(peakHeap, process.memoryUsage().heapUsed); + }; + const registry = registryFor(scenario, { + sample, + beforeReturn: async () => { + storedLive = await directorySize(directory); + }, + }); + const baseHeap = heapAfterGc(); + + const storage = await open(scenario.backend, directory); + let commits = 0; + const written = noWrites(); + const commit = storage.commit.bind(storage); + storage.commit = (writes, commitContext) => { + commits++; + // SQLite commits synchronously, which can starve the timer; sample here too. + sample(); + for (const write of writes) written[write.type] += JSON.stringify(write).length; + return commit(writes, commitContext); + }; + const harness = await Harness.open(storage, { models, registry }, context); + const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); + faux.setResponses([ + fauxAssistantMessage([fauxToolCall("script", { count: scenario.count }, { id: "s" })], { + stopReason: "toolUse", + }), + fauxAssistantMessage([fauxText("done")]), + ]); + const before = { commits, written: { ...written } }; + peakHeap = 0; + const sampler = setInterval(sample, 5); + const started = performance.now(); + const settled = await (await root.submit({ type: "input", content: "run" }, context)).wait(context); + if (settled.status !== "done") throw new Error(`run: ${settled.status}`); + // `done` only says the model answered; the script's result says whether it ran through. + const latest = (await root.entries({}, 10, undefined, context)).items; + const result = latest.find((entry) => ToolResultEntry.is(entry))?.model?.[0]; + if (result?.role !== "toolResult" || result.isError) throw new Error("run: the script failed"); + const ms = Math.round(performance.now() - started); + clearInterval(sampler); + const settledHeap = heapAfterGc() - baseHeap; + await harness.waitForIdle(context); + const idleHeap = heapAfterGc() - baseHeap; + const delta = noWrites(); + for (const type of Object.keys(delta) as StorageWrite["type"][]) { + delta[type] = written[type] - before.written[type]; + } + const storedRun = await directorySize(directory); + await harness.close(context); + const storedClosed = await directorySize(directory); + + const reopenedStorage = await open(scenario.backend, directory); + const reopened = await Harness.open(reopenedStorage, { models, registry }, context); + const reopenedHeap = heapAfterGc() - baseHeap; + const storedReopened = await directorySize(directory); + let nestedTasks = 0; + let largestTask = 0; + let cursor: Parameters[2]; + do { + const page = await reopenedStorage.scanTasks({ kind: "pi.tool" }, 50, cursor, context); + for (const record of page.items) { + if ((record.input as ToolTaskInput).kind === "nested") nestedTasks++; + largestTask = Math.max(largestTask, JSON.stringify(record).length); + } + cursor = page.next; + } while (cursor !== undefined); + await reopened.close(context); + return { + commits: commits - before.commits, + ms, + written: delta, + storedLive, + storedRun, + storedClosed, + storedReopened, + peakHeap: peakHeap - baseHeap, + settledHeap, + idleHeap, + reopenedHeap, + nestedTasks, + largestTask, + }; + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +function mib(bytes: number): string { + return `${(bytes / 1024 / 1024).toFixed(1)} MiB`; +} + +function kib(bytes: number): string { + return `${(bytes / 1024).toFixed(1)} KiB`; +} + +async function main(): Promise { + const counts = process.argv[2] === undefined ? [50, 200] : [Number(process.argv[2])]; + const bytes = 1024 * 1024; + console.log(`Images: ${bytes / 1024} KiB base64 each, read then classified sequentially in one script call`); + console.log( + "backend | variant | count | commits, ms | written: all (task / doc create / doc change / entry) | stored: live / run / closed / reopened | heap: peak; settled / idle / reopened | nested tasks, largest task", + ); + for (const count of counts) { + for (const backend of ["sqlite", "jsonl"] as const) { + for (const variant of ["inline", "reference"] as const) { + const result = await runScenario({ backend, variant, count, bytes }); + const all = Object.values(result.written).reduce((sum, value) => sum + value, 0); + const written = result.written; + console.log( + [ + backend.padEnd(7), + variant.padEnd(9), + String(count).padEnd(5), + `${result.commits}, ${result.ms} ms`, + `${mib(all)} (${mib(written.task)} / ${mib(written["document.create"])} / ${mib(written["document.change"])} / ${mib(written.entry)})`, + `${mib(result.storedLive)} / ${mib(result.storedRun)} / ${mib(result.storedClosed)} / ${mib(result.storedReopened)}`, + `${mib(result.peakHeap)}; ${mib(result.settledHeap)} / ${mib(result.idleHeap)} / ${mib(result.reopenedHeap)}`, + `${result.nestedTasks}, ${kib(result.largestTask)}`, + ].join(" | "), + ); + } + } + } +} + +await main(); diff --git a/packages/durable/test/nested-tools-bench.ts b/packages/durable/test/nested-tools-bench.ts new file mode 100644 index 000000000..b3af3c61c --- /dev/null +++ b/packages/durable/test/nested-tools-bench.ts @@ -0,0 +1,253 @@ +/** + * Nested tool call storage benchmark. Per run, one model-issued tool makes `calls` parallel nested calls of `blob`, + * each streaming `bytes` of output in 8 chunks `gapMs` apart; at depth 2 it makes them through `calls / 10` parallel + * `fan` calls of 10 each. With `progress: false` the nested calls commit no running output. Reports commits, logical + * bytes written (JSON of the storage writes) by write type, the stored size after each run, and the heap: its peak during a run, sampled every 5 ms, and what stays after a + * run and a GC, both over the heap after a GC before the first run. A 2 ms gap is a fast tool such as `read`: it ends + * before a second progress commit. A 50 ms gap streams for 400 ms, as a short `bash` command does, and commits progress + * every 100 ms. + * + * node --conditions=source --experimental-strip-types --expose-gc test/nested-tools-bench.ts [streaming|nesting] + */ +import { mkdtemp, readdir, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { BACKGROUND_CONTEXT as context } from "@earendil-works/chord/context"; +import { createModels } from "@earendil-works/pi-ai/models"; +import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; +import { Type } from "typebox"; +import { + type Conversation, + createRegistry, + defineExtension, + defineTool, + Harness, + type Storage, + type StorageWrite, + ToolResultEntry, +} from "../src/index.ts"; +import { openNodeJsonlStorage } from "../src/storage/jsonl/node.ts"; +import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; + +type Backend = "sqlite" | "jsonl"; +type Scenario = { + backend: Backend; + calls: number; + depth: 1 | 2; + bytes: number; + gapMs: number; + /** Whether nested calls stream their running output to `pi.live`. */ + progress: boolean; + runs: number; +}; +type Written = Record; +type Run = { + commits: number; + written: Written; + storedBytes: number; + ms: number; + /** Over the heap after a GC before the first run. */ + peakHeap: number; + heldHeap: number; +}; + +const gc = (globalThis as { gc?: () => void }).gc; +if (gc === undefined) throw new Error("Run with --expose-gc"); + +function heapAfterGc(): number { + gc!(); + return process.memoryUsage().heapUsed; +} + +const CHUNKS = 8; +/** Calls each inner `fan` makes at depth 2. */ +const FAN_OUT = 10; + +async function directorySize(path: string): Promise { + let total = 0; + for (const entry of await readdir(path, { withFileTypes: true })) { + const full = join(path, entry.name); + total += entry.isDirectory() ? await directorySize(full) : (await stat(full)).size; + } + return total; +} + +async function open(backend: Backend, directory: string): Promise { + return backend === "sqlite" + ? openNodeSqliteStorage(join(directory, "session.sqlite")) + : openNodeJsonlStorage(directory, context); +} + +function noWrites(): Written { + return { + conversation: 0, + entry: 0, + task: 0, + submission: 0, + "document.create": 0, + "document.copy": 0, + "document.change": 0, + "document.retire": 0, + }; +} + +async function runScenario(scenario: Scenario): Promise<{ perRun: Run[]; closedBytes: number }> { + const directory = await mkdtemp(join(tmpdir(), "pi-durable-nested-bench-")); + try { + const faux = fauxProvider(); + const models = createModels(); + models.setProvider(faux.provider); + const registry = createRegistry(); + const chunk = `${"x".repeat(scenario.bytes / CHUNKS - 1)}\n`; + // Streams its output and returns none, so its result is the retained output, as bash's is. + const blob = defineTool({ + name: "blob", + description: "Stream a blob", + parameters: Type.Object({ n: Type.Number() }), + execute: async (_args, api) => { + for (let n = 0; n < CHUNKS; n++) { + api.output(chunk); + await new Promise((resolve) => setTimeout(resolve, scenario.gapMs)); + } + return {}; + }, + }); + const options = scenario.progress ? {} : { progress: false as const }; + /** A tool that makes `count` parallel nested calls of `target`, which get `{ n, count: FAN_OUT }`. */ + const fanOut = (name: string, target: string) => + defineTool({ + name, + description: `Call ${target} in parallel`, + parameters: Type.Object({ count: Type.Number() }), + execute: async (args, api, callContext) => { + const calls = Array.from({ length: args.count }, (_, n) => + api.executeTool(target, { n, count: FAN_OUT }, callContext, options), + ); + const failed = (await Promise.all(calls)).filter((result) => result.isError).length; + return { output: [{ type: "text", text: `${failed} failed` }], isError: failed > 0 }; + }, + }); + registry.install(defineExtension({ name: "bench", tools: [blob, fanOut("fan", "blob"), fanOut("fan2", "fan")] })); + + const storage = await open(scenario.backend, directory); + let commits = 0; + const written = noWrites(); + const commit = storage.commit.bind(storage); + storage.commit = (writes, commitContext) => { + commits++; + for (const write of writes) written[write.type] += JSON.stringify(write).length; + return commit(writes, commitContext); + }; + const harness = await Harness.open(storage, { models, registry }, context); + const root = await harness.root(context, { agent: { model: { provider: "faux", modelId: "faux-1" } } }); + const perRun: Run[] = []; + const top = scenario.depth === 1 ? scenario.calls : scenario.calls / FAN_OUT; + const baseHeap = heapAfterGc(); + for (let run = 0; run < scenario.runs; run++) { + faux.setResponses([ + fauxAssistantMessage( + [fauxToolCall(scenario.depth === 1 ? "fan" : "fan2", { count: top }, { id: `r${run}` })], + { + stopReason: "toolUse", + }, + ), + fauxAssistantMessage([fauxText("done")]), + ]); + const before = { commits, written: { ...written } }; + let peakHeap = 0; + const sampler = setInterval(() => { + peakHeap = Math.max(peakHeap, process.memoryUsage().heapUsed); + }, 5); + const started = performance.now(); + const settled = await (await root.submit({ type: "input", content: `run ${run}` }, context)).wait(context); + if (settled.status !== "done") throw new Error(`run ${run}: ${settled.status}`); + await assertToolsSucceeded(root, run); + await harness.waitForIdle(context); + const ms = Math.round(performance.now() - started); + clearInterval(sampler); + const delta = noWrites(); + for (const type of Object.keys(delta) as StorageWrite["type"][]) { + delta[type] = written[type] - before.written[type]; + } + perRun.push({ + commits: commits - before.commits, + written: delta, + storedBytes: await directorySize(directory), + ms, + peakHeap: peakHeap - baseHeap, + heldHeap: heapAfterGc() - baseHeap, + }); + } + await harness.close(context); + return { perRun, closedBytes: await directorySize(directory) }; + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +/** A run that answers `done` may still have failed tools; the model-issued call's result says whether any did. */ +async function assertToolsSucceeded(root: Conversation, run: number): Promise { + const page = await root.entries({}, 10, undefined, context); + const result = page.items.find((entry) => ToolResultEntry.is(entry))?.model?.[0]; + if (result?.role !== "toolResult" || result.isError) throw new Error(`run ${run}: a nested call failed`); +} + +function mib(bytes: number): string { + return `${(bytes / 1024 / 1024).toFixed(2)} MiB`; +} + +async function report(title: string, scenarios: Scenario[]): Promise { + console.log(`\n${title}`); + console.log( + "backend | calls | depth | output | gap | stream | run 1: commits, ms, written (all; tasks / doc changes / doc creates) | stored after run 1 / 2 / 3 | after close | heap: run 1 peak; held after run 1 / 2 / 3", + ); + for (const scenario of scenarios) { + const { perRun, closedBytes } = await runScenario(scenario); + const first = perRun[0]!; + const all = Object.values(first.written).reduce((sum, bytes) => sum + bytes, 0); + console.log( + [ + scenario.backend.padEnd(6), + String(scenario.calls).padEnd(5), + String(scenario.depth).padEnd(5), + `${scenario.bytes / 1024} KiB`.padEnd(6), + `${scenario.gapMs} ms`.padEnd(5), + (scenario.progress ? "yes" : "no").padEnd(6), + `${first.commits}, ${first.ms} ms, ${mib(all)} (${mib(first.written.task)} / ${mib(first.written["document.change"])} / ${mib(first.written["document.create"])})`, + perRun.map((run) => mib(run.storedBytes)).join(" / "), + mib(closedBytes), + `${mib(first.peakHeap)}; ${perRun.map((run) => mib(run.heldHeap)).join(" / ")}`, + ].join(" | "), + ); + } +} + +async function main(): Promise { + const section = process.argv[2]; + if (section === undefined || section === "streaming") { + const scenarios: Scenario[] = []; + for (const backend of ["sqlite", "jsonl"] as const) { + for (const bytes of [1024, 16 * 1024]) { + for (const gapMs of [2, 50]) { + for (const progress of [true, false]) { + scenarios.push({ backend, calls: 200, depth: 1, bytes, gapMs, progress, runs: 3 }); + } + } + } + } + await report("Streaming: 200 calls, with and without progress", scenarios); + } + if (section === undefined || section === "nesting") { + const scenarios: Scenario[] = []; + for (const backend of ["sqlite", "jsonl"] as const) { + for (const calls of [200, 2000]) { + for (const depth of [1, 2] as const) { + scenarios.push({ backend, calls, depth, bytes: 1024, gapMs: 2, progress: false, runs: 3 }); + } + } + } + await report("Nesting: fast calls without progress, at depth 1 and 2", scenarios); + } +} + +await main(); diff --git a/packages/durable/test/session-documents.test.ts b/packages/durable/test/session-documents.test.ts index c9764eac7..293778b02 100644 --- a/packages/durable/test/session-documents.test.ts +++ b/packages/durable/test/session-documents.test.ts @@ -437,7 +437,7 @@ describe("Session document transactions", () => { expect((await session.snapshot(CounterDoc, context))!.count).toBe(3); }); - it("poisons the Session after an uncertain Storage failure and publishes nothing", async () => { + it("fails the Session after a Storage failure and publishes nothing", async () => { const { session, storage, publications, conversationId } = await setupLive(); await flush(); const before = (await session.snapshot(LiveDoc, conversationId, context))!; @@ -451,8 +451,8 @@ describe("Session document transactions", () => { await flush(); expect(publications.length).toBe(published); expect(before.message).toBeUndefined(); - await expect(session.snapshot(LiveDoc, conversationId, context)).rejects.toThrow("poisoned"); - await expect(session.commit(() => undefined, context)).rejects.toThrow("poisoned"); + await expect(session.snapshot(LiveDoc, conversationId, context)).rejects.toMatchObject({ name: "SessionFailed" }); + await expect(session.commit(() => undefined, context)).rejects.toMatchObject({ name: "SessionFailed" }); await session.close(context); }); diff --git a/packages/durable/test/session-forks.test.ts b/packages/durable/test/session-forks.test.ts index 76400af22..cef7c7938 100644 --- a/packages/durable/test/session-forks.test.ts +++ b/packages/durable/test/session-forks.test.ts @@ -4,7 +4,6 @@ import { defineDocFamily, defineTask, type EntryId, - StorageRejected, type StorageWrite, type TaskId, } from "@earendil-works/pi-durable"; @@ -550,7 +549,7 @@ describe("Session conversation document forks", () => { expect(await session.snapshot(Copied, parentId, context)).toEqual({ value: "copied" }); }); - it("rolls back a guaranteed Storage rejection without poisoning the Session", async () => { + it("fails the Session on a rejected fork commit, which leaves no effect", async () => { const Doc = defineDoc<{ value: string }>({ kind: "fork.storage-rejected", version: 1, @@ -567,15 +566,19 @@ describe("Session conversation document forks", () => { await tx.doc(Doc, parentId); }, context); let rejectedChildId!: ConversationId; - storage.failNextCommit(new StorageRejected("copy rejected")); + const rejection = new Error("copy rejected"); + storage.failNextCommit(rejection); await expect( session.commit(async (tx) => { rejectedChildId = (await tx.forkConversation(parentId, forkAt, { ownership: { kind: "ownerless" } })).id; }, context), - ).rejects.toThrow("copy rejected"); - expect(await storage.conversation(rejectedChildId, context)).toBeUndefined(); - const next = await session.commit((tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), context); - expect(await storage.conversation(next.id, context)).toEqual(next); + ).rejects.toBe(rejection); + await expect( + session.commit((tx) => tx.createConversation({ ownership: { kind: "ownerless" } }), context), + ).rejects.toMatchObject({ name: "SessionFailed", cause: rejection }); + // The failed Session closed its Storage; reopened, it holds nothing of the batch. + expect(await session.closed).toEqual({ reason: "failed", error: rejection }); + expect(await storage.reopen().conversation(rejectedChildId, context)).toBeUndefined(); }); it("retires a copied document and can recreate the address in the fork transaction", async () => { diff --git a/packages/durable/test/session-support.ts b/packages/durable/test/session-support.ts index e8da108d8..99a13a3ab 100644 --- a/packages/durable/test/session-support.ts +++ b/packages/durable/test/session-support.ts @@ -66,6 +66,12 @@ export class ControlledStorage extends MemoryStorage { this.#commitFailure = error; } + /** Reopen after `close()`, as a fresh process would reopen the same database, keeping every committed record. */ + reopen(): this { + (this as unknown as { closed: boolean }).closed = false; + return this; + } + #release(kind: "commit" | "find", held: { gate: Deferred }): void { if (kind === "commit" && this.#commitGate === held) this.#commitGate = undefined; if (kind === "find" && this.#findGate === held) this.#findGate = undefined; diff --git a/packages/durable/test/spec-usage.test.ts b/packages/durable/test/spec-usage.test.ts index a38161a4e..7ed3478b5 100644 --- a/packages/durable/test/spec-usage.test.ts +++ b/packages/durable/test/spec-usage.test.ts @@ -181,7 +181,7 @@ function examples() { const settled = await (await handle.submit(request, context)).wait(context); if (settled.status !== "done" || settled.type !== "input") throw new Error(`Subagent failed: ${settled.status}`); - return { content: [{ type: "text", text: await answerText(api, settled.answer, context) }] }; + return { output: [{ type: "text", text: await answerText(api, settled.answer, context) }] }; }, }), ], diff --git a/packages/durable/test/sqlite-facade.test.ts b/packages/durable/test/sqlite-facade.test.ts index 86b2bb096..c3db89910 100644 --- a/packages/durable/test/sqlite-facade.test.ts +++ b/packages/durable/test/sqlite-facade.test.ts @@ -1,7 +1,6 @@ import { DatabaseSync, type StatementSync } from "node:sqlite"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import { describe, expect, it } from "vitest"; -import { StorageRejected } from "../src/errors.ts"; import { idFromNumber } from "../src/ids.ts"; import type { SqliteDatabase, SqliteExecutor, SqliteValue } from "../src/storage/sqlite/index.ts"; import { SqliteStorage } from "../src/storage/sqlite/index.ts"; @@ -278,14 +277,14 @@ describe("portable SQLite facade settlement", () => { await database.close(); }); - it("does not preserve a guaranteed rejection when rollback itself fails", async () => { + it("does not preserve the callback error when rollback itself fails", async () => { const database = await openNodeSqliteDatabase(":memory:"); await database.exec("CREATE TABLE rollback_probe (value INTEGER)"); await expect( database.transaction(async (transaction) => { await transaction.exec("INSERT INTO rollback_probe (value) VALUES (1)"); await transaction.exec("COMMIT"); - throw new StorageRejected("rejected after an escaped commit"); + throw new Error("rejected after an escaped commit"); }), ).rejects.toThrow(AggregateError); expect(await database.get("SELECT value FROM rollback_probe")).toEqual({ value: 1 }); diff --git a/packages/durable/test/system-order-cache-e2e.test.ts b/packages/durable/test/system-order-cache-e2e.test.ts index e4ed24b6f..0d67629d5 100644 --- a/packages/durable/test/system-order-cache-e2e.test.ts +++ b/packages/durable/test/system-order-cache-e2e.test.ts @@ -46,13 +46,13 @@ const first = defineTool({ name: "probe_first", description: "Unused probe tool.", parameters: Parameters, - execute: async () => ({ content: [] }), + execute: async () => ({ output: [] }), }); const second = defineTool({ name: "probe_second", description: "Unused probe tool added on the second turn.", parameters: Parameters, - execute: async () => ({ content: [] }), + execute: async () => ({ output: [] }), }); /** Puts the leading system message back after the user messages before it: the request order before the fix. */ diff --git a/packages/durable/test/tools-read-differential.test.ts b/packages/durable/test/tools-read-differential.test.ts index c191d8a1f..ea846f7d1 100644 --- a/packages/durable/test/tools-read-differential.test.ts +++ b/packages/durable/test/tools-read-differential.test.ts @@ -18,7 +18,7 @@ function referenceRead(bytes: Uint8Array, path: string, offset: number | undefin const mimeType = detectSupportedImageMimeType(bytes); if (mimeType) { return { - content: [], + output: [], isError: true, diagnostics: [ { @@ -79,7 +79,7 @@ function referenceRead(bytes: Uint8Array, path: string, offset: number | undefin }); } return { - content: outputText === "" ? [] : [{ type: "text", text: outputText }], + output: outputText === "" ? [] : [{ type: "text", text: outputText }], ...(details === undefined ? {} : { details }), diagnostics, }; diff --git a/packages/durable/test/tools.test.ts b/packages/durable/test/tools.test.ts index 5dcdd9f12..d18d728d7 100644 --- a/packages/durable/test/tools.test.ts +++ b/packages/durable/test/tools.test.ts @@ -70,6 +70,7 @@ function fakeApi(env: ExecutionEnv | undefined): { env, output: (chunk: string | Uint8Array) => output.push(typeof chunk === "string" ? chunk : new TextDecoder().decode(chunk)), + retainedOutput: () => ({ text: output.join(""), truncated: false }), diagnostic: (diagnostic: ToolDiagnostic) => diagnostics.push(diagnostic), details: async () => {}, } as unknown as ToolExecutionApi; @@ -81,9 +82,9 @@ async function run( args: JsonValue, env: ExecutionEnv | undefined, context: Context = BACKGROUND_CONTEXT, -): Promise { +): Promise { const { api, output, diagnostics } = fakeApi(env); - return { ...(await tool.execute(args, api, context)), output, reported: diagnostics }; + return { ...(await tool.execute(args, api, context)), streamed: output, reported: diagnostics }; } /** Run a tool expected to throw; returns the error with what it streamed and reported first. */ @@ -91,18 +92,18 @@ async function runFailing( tool: ToolRegistration, args: JsonValue, env: ExecutionEnv, -): Promise<{ error: Error; output: string[]; reported: ToolDiagnostic[] }> { +): Promise<{ error: Error; streamed: string[]; reported: ToolDiagnostic[] }> { const { api, output, diagnostics } = fakeApi(env); try { await tool.execute(args, api, BACKGROUND_CONTEXT); } catch (error) { - return { error: error as Error, output, reported: diagnostics }; + return { error: error as Error, streamed: output, reported: diagnostics }; } throw new Error("Expected the tool to throw"); } function textOutput(result: ToolExecutionResult): string { - return (result.content ?? []).flatMap((part) => (part.type === "text" ? [part.text] : [])).join("\n"); + return (result.output ?? []).flatMap((part) => (part.type === "text" ? [part.text] : [])).join("\n"); } function diagnosticText(result: ToolExecutionResult): string { @@ -312,7 +313,7 @@ describe("durable tools", () => { ); getOrThrow(await env.writeFile("image.txt", png, BACKGROUND_CONTEXT)); const result = await run(createReadTool(), { path: "image.txt" }, env); - expect(result).toMatchObject({ content: [], isError: true }); + expect(result).toMatchObject({ output: [], isError: true }); expect(diagnosticText(result)).toBe("image.txt is an image (image/png); reading images is not supported"); }); }); @@ -570,7 +571,7 @@ describe("durable tools", () => { it("runs the command with pwsh as one argument, forcing UTF-8 output", async () => { const { env, commands } = programsEnv(["pwsh"], "héllo\n"); const result = await run(createPowerShellTool(), { command: "Write-Output 'héllo'" }, env); - expect(result.output.join("")).toBe("héllo\n"); + expect(result.streamed.join("")).toBe("héllo\n"); expect(commands).toEqual([ [ "pwsh", @@ -591,7 +592,7 @@ describe("durable tools", () => { { command: "$x" }, windowsOnly.env, ); - expect(result.output.join("")).toBe("ok"); + expect(result.streamed.join("")).toBe("ok"); expect(windowsOnly.commands.map((command) => command[0])).toEqual(["pwsh", "powershell"]); expect(String(windowsOnly.commands[1]?.at(-1))).toMatch(/\n\$x = 1\n\$x$/); @@ -600,11 +601,15 @@ describe("durable tools", () => { expect(failed.error.message).toBe("spawn powershell ENOENT"); }); - it("throws on a nonzero exit after streaming the output", async () => { + it("answers a nonzero exit with an error result carrying the exit code", async () => { const { env } = programsEnv(["pwsh"], "partial", 3); - const failed = await runFailing(createPowerShellTool(), { command: "exit 3" }, env); - expect(failed.error.message).toBe("Command exited with code 3"); - expect(failed.output.join("")).toBe("partial"); + const result = await run(createPowerShellTool(), { command: "exit 3" }, env); + expect(result).toMatchObject({ + isError: true, + structuredOutput: { output: "partial", truncated: false, exitCode: 3 }, + diagnostics: [{ severity: "error", code: "exit_code", message: "Command exited with code 3" }], + }); + expect(result.streamed.join("")).toBe("partial"); }); it.runIf(process.platform === "win32")("runs real PowerShell with UTF-8 output", async () => { @@ -613,7 +618,7 @@ describe("durable tools", () => { { command: "Write-Output ('h' + [char]0xe9 + 'llo'); exit 0" }, createEnv(), ); - expect(result.output.join("").trim()).toBe("héllo"); + expect(result.streamed.join("").trim()).toBe("héllo"); }); }); @@ -644,19 +649,24 @@ describe("durable tools", () => { expect(calls).toEqual([["tail\n", skipped]]); }); - it("streams combined stdout and stderr and returns no content of its own", async () => { + it("streams combined stdout and stderr, returns no output of its own, and repeats it with the exit code", async () => { const result = await run(createBashTool(), { command: "printf out; printf err >&2" }, createEnv()); - expect(result.output.join("")).toContain("out"); - expect(result.output.join("")).toContain("err"); - expect(result.content).toBeUndefined(); + expect(result.streamed.join("")).toContain("out"); + expect(result.streamed.join("")).toContain("err"); + expect(result.output).toBeUndefined(); + expect(result.structuredOutput).toEqual({ output: result.streamed.join(""), truncated: false, exitCode: 0 }); }); - it("throws on nonzero exits and timeouts after streaming the output", async () => { + it("answers a nonzero exit with an error result, and throws on a timeout, after streaming the output", async () => { const env = createEnv(); const tool = createBashTool(); - const failed = await runFailing(tool, { command: "printf failed; exit 7" }, env); - expect(failed.error.message).toBe("Command exited with code 7"); - expect(failed.output.join("")).toBe("failed"); + const failed = await run(tool, { command: "printf failed; exit 7" }, env); + expect(failed).toMatchObject({ + isError: true, + structuredOutput: { output: "failed", truncated: false, exitCode: 7 }, + diagnostics: [{ code: "exit_code", message: "Command exited with code 7" }], + }); + expect(failed.streamed.join("")).toBe("failed"); const slow = await runFailing(tool, { command: "sleep 2", timeout: 0.01 }, env); expect(slow.error.message).toBe("Command timed out after 0.01 seconds"); }); @@ -706,7 +716,7 @@ describe("durable tools", () => { process.platform === "win32" ? "" : `:${getOrThrow(await env.canonicalPath(workspace, BACKGROUND_CONTEXT))}`; - expect(result.output.join("")).toBe(`ready::explicit${pwd}`); + expect(result.streamed.join("")).toBe(`ready::explicit${pwd}`); expect(getOrThrow(await env.exists(`${workspace}/prepared-cwd`, BACKGROUND_CONTEXT))).toBe(true); }); @@ -716,7 +726,7 @@ describe("durable tools", () => { { command: "printf $value" }, createEnv(), ); - expect(result.output.join("")).toBe("hello"); + expect(result.streamed.join("")).toBe("hello"); }); it("streams every byte and spills complete output beyond the default limits", async () => { @@ -727,7 +737,7 @@ describe("durable tools", () => { env, ); const expected = Array.from({ length: 3000 }, (_, index) => `line-${index + 1}\n`).join(""); - expect(result.output.join("")).toBe(expected); + expect(result.streamed.join("")).toBe(expected); const fullOutputPath = result.reported[0]?.message.match(/^Full output: (.+)$/)?.[1]; expect(fullOutputPath).toBeDefined(); expect(getOrThrow(await env.readTextFile(fullOutputPath!, BACKGROUND_CONTEXT))).toBe(expected); diff --git a/scripts/issues/triage.ts b/scripts/issues/triage.ts index 2ee0fbd38..9142f1e1d 100644 --- a/scripts/issues/triage.ts +++ b/scripts/issues/triage.ts @@ -290,7 +290,7 @@ export class Triage { replay: "safe", execute: async (args, api) => { const hits = corpus.search(args.query, issueFor(api.conversationId)); - return { content: [{ type: "text", text: hits.length ? hits.map(line).join("\n") : "No matches." }] }; + return { output: [{ type: "text", text: hits.length ? hits.map(line).join("\n") : "No matches." }] }; }, }); @@ -301,9 +301,9 @@ export class Triage { replay: "safe", execute: async (args) => { const issue = corpus.get(args.number); - if (!issue) return { content: [{ type: "text", text: `#${args.number} is not in the local corpus.` }] }; + if (!issue) return { output: [{ type: "text", text: `#${args.number} is not in the local corpus.` }] }; const text = `#${issue.number} [${issueStatus(issue)}] ${issue.title}\nLabels: ${issue.labels.join(", ") || "none"}. Created ${issue.createdAt}.\n\n${issue.body.slice(0, 8000)}`; - return { content: [{ type: "text", text }] }; + return { output: [{ type: "text", text }] }; }, }); @@ -329,7 +329,7 @@ export class Triage { replay: "safe", execute: async (args, api, callContext) => { const number = issueFor(api.conversationId); - if (number < 0) return { content: [{ type: "text", text: "Unknown conversation." }], isError: true }; + if (number < 0) return { output: [{ type: "text", text: "Unknown conversation." }], isError: true }; const result: TriageResult = { ...args, duplicates: args.duplicates.filter((dup) => dup.number !== number).slice(0, 5), @@ -344,7 +344,7 @@ export class Triage { }), callContext, ); - return { content: [{ type: "text", text: "Recorded." }], control: { terminate: true } }; + return { output: [{ type: "text", text: "Recorded." }], control: { terminate: true } }; }, });