feat(agent): add hardened pico3 kernel

This commit is contained in:
Mario Zechner 2026-09-14 11:17:04 +02:00
parent 9b2aff2c5b
commit 46b66c59af
51 changed files with 15148 additions and 1 deletions

View file

@ -291,7 +291,7 @@ The captured transcript must retain all of the following:
5. every entry's edits verbatim; the view is the transcript, not the edited/model-message projection;
6. every later committed entry for the lifetime of the watch—there is no live cutoff.
When a committed entry carries a new head, `applyDelta` truncates the existing active transcript using this same fork-aware boundary definition and then appends the new entry at its chronological position. It must not apply context derivation's special treatment of heads. A fresh capture and a client that folded every committed delta must produce identical `entries`.
When a committed entry carries a new head, `applyEnvelope` truncates the existing active transcript using this same fork-aware boundary definition and then appends the new entry at its chronological position. It must not apply context derivation's special treatment of heads. A fresh capture and a client that folded every committed envelope must produce identical `entries`.
Add a regression with a child conversation whose newest head points to an entry in its parent and with an older head entry still inside that retained range. Assert that capture includes the inherited target, preserves both heads in chronological order, and equals the result of incremental folding.
@ -322,6 +322,13 @@ JSONL multi-file publication:
Add fault-injection tests for a crash after each sidecar append and before/after the main marker. Graceful `Harness.close()` is not a storage crash test.
Durability modes and failure boundary:
- `fsync: false` covers callback failures, append failures, and process termination while the OS/filesystem remain alive. Complete published records recover; torn final lines are truncated; unconfirmed sidecar tails roll back.
- `fsync: false` does **not** promise that a returned commit survives power loss, kernel/VM-host failure, storage-cache loss, or a filesystem that loses or reorders completed writes.
- Worst case without fsync: acknowledged tail commits disappear. If sidecars survive without their marker, replay rolls them back. If a marker survives without a required sidecar, open fails rather than exposing a half-commit. If the lost commit was a pre-effect tool/job checkpoint or memo, recovery may attempt the external effect again; external idempotency is still required.
- `fsync: true` flushes each sidecar before flushing the main marker, strengthening file-data durability and ordering. It is not a complete database guarantee until parent-directory metadata for file creation, rename, and unlink is also synchronized, and it cannot strengthen a filesystem or device that provides weaker guarantees.
### 11. Terminal task retention
Remove `pruneTerminal` behavior that makes terminal tasks unreadable. `getTask`, `waitForTask`, task scans, dependencies, and typed outcomes must continue to work after arbitrary later activity and reopen.

View file

@ -29,6 +29,13 @@ Read order: §1–§5 are the harness alone (no Chord); §6 is the Chord layer;
namespace state (caches, keys, secrets) and slot working state (memos,
idempotency evidence) never appear in the view.
Here, “durable” means recoverable according to the selected storage mode. JSONL
with `fsync: false` covers process termination while the OS/filesystem remain
alive, but acknowledged tail commits may disappear after power, kernel, VM-host,
or storage-cache failure. `fsync: true` strengthens file-data durability; see
`hardening-handoff.md` §10 for the exact boundary and worst cases. Neither mode
makes an external effect exactly once without an external idempotency key.
## 2. Durable primitives
| primitive | lives in | lifetime | written by | forks / history | in the view |

View file

@ -18,6 +18,10 @@
"types": "./dist/harness/context.d.ts",
"import": "./dist/harness/context.js"
},
"./experimental/pico3": {
"types": "./dist/harness/pico3/index.d.ts",
"import": "./dist/harness/pico3/index.js"
},
"./harness/env/nodejs": {
"types": "./dist/harness/env/nodejs.d.ts",
"import": "./dist/harness/env/nodejs.js"

View file

@ -0,0 +1,29 @@
import { spawn } from "node:child_process";
import { Type } from "typebox";
import type { ToolDeclaration, ToolResult } from "./types.ts";
const parameters = Type.Object({ command: Type.String(), cwd: Type.Optional(Type.String()) });
/** Run a shell command. Output is piped to the kernel; bounds come from `output`. */
export function bashTool(output: ToolDeclaration["output"] = {}): ToolDeclaration<typeof parameters> {
return {
name: "bash",
description: "Run a shell command",
parameters,
replay: "unsafe",
output,
async execute({ command, cwd }, api, ctx): Promise<ToolResult> {
const started = Date.now();
const child = spawn("bash", ["-c", command], { cwd, stdio: ["ignore", "pipe", "pipe"] });
const onAbort = () => child.kill("SIGKILL");
ctx.abortSignal?.addEventListener("abort", onAbort);
child.stdout.on("data", (c: Uint8Array) => api.stream(c));
child.stderr.on("data", (c: Uint8Array) => api.stream(c));
const { code, signal } = await new Promise<{ code: number | null; signal: string | null }>((r) =>
child.on("close", (code, signal) => r({ code, signal })),
);
ctx.abortSignal?.removeEventListener("abort", onAbort);
return { isError: code !== 0, details: { exitCode: code, signal, ms: Date.now() - started } };
},
};
}

View file

@ -0,0 +1,100 @@
/**
* Bounded byte collector. It retains a prefix or suffix constrained by both a byte
* budget and a newline budget, and accounts for every discarded byte/newline.
*/
export class Bounded {
private bytes: Uint8Array<ArrayBufferLike> = new Uint8Array();
private readonly maxBytes: number;
private readonly maxLines: number;
private readonly retain: "head" | "tail";
droppedBytes = 0;
droppedLines = 0;
total = 0;
constructor(maxBytes: number, maxLines: number, retain: "head" | "tail") {
this.maxBytes = Math.max(0, maxBytes);
this.maxLines = Math.max(0, maxLines);
this.retain = retain;
}
push(chunk: Uint8Array): void {
this.total += chunk.length;
if (chunk.length === 0) return;
if (this.maxBytes === 0 || this.maxLines === 0) {
this.drop(chunk);
return;
}
if (this.retain === "head") {
this.pushHead(chunk);
return;
}
this.pushTail(chunk);
}
private pushHead(chunk: Uint8Array): void {
const remainingBytes = this.maxBytes - this.bytes.length;
const remainingLines = this.maxLines - countNewlines(this.bytes);
if (remainingBytes <= 0 || remainingLines <= 0) {
this.drop(chunk);
return;
}
let take = Math.min(chunk.length, remainingBytes);
let lines = 0;
for (let i = 0; i < take; i++) {
if (chunk[i] !== 0x0a) continue;
lines++;
if (lines === remainingLines) {
take = i + 1;
break;
}
}
this.bytes = concat(this.bytes, chunk.subarray(0, take));
this.drop(chunk.subarray(take));
}
private pushTail(chunk: Uint8Array): void {
const incomingStart = tailStart(chunk, this.maxBytes, this.maxLines);
this.drop(chunk.subarray(0, incomingStart));
const combined = concat(this.bytes, chunk.subarray(incomingStart));
const start = tailStart(combined, this.maxBytes, this.maxLines);
this.drop(combined.subarray(0, start));
this.bytes = combined.slice(start);
}
private drop(bytes: Uint8Array): void {
this.droppedBytes += bytes.length;
this.droppedLines += countNewlines(bytes);
}
get dropped(): number {
return this.droppedBytes;
}
text(): string {
return new TextDecoder().decode(this.bytes);
}
}
function tailStart(bytes: Uint8Array, maxBytes: number, maxLines: number): number {
let start = Math.max(0, bytes.length - maxBytes);
let excessLines = countNewlines(bytes.subarray(start)) - maxLines;
for (; start < bytes.length && excessLines > 0; start++) {
if (bytes[start] === 0x0a) excessLines--;
}
return start;
}
function concat(a: Uint8Array, b: Uint8Array): Uint8Array {
if (a.length === 0) return b.slice();
if (b.length === 0) return a;
const out = new Uint8Array(a.length + b.length);
out.set(a);
out.set(b, a.length);
return out;
}
function countNewlines(bytes: Uint8Array): number {
let count = 0;
for (const byte of bytes) if (byte === 0x0a) count++;
return count;
}

View file

@ -0,0 +1,178 @@
import { type Context, defineService, type MutableReplicatedState, type ReplicatedState } from "@earendil-works/chord";
import type { Op, Path, Seg } from "@earendil-works/chord/delta";
import type { ConversationHandle, Harness } from "./harness.ts";
import type {
ConversationSpec,
ConversationView,
Entry,
EntryScan,
Envelope,
Id,
JsonObject,
NewEntry,
SendInput,
ViewEvent,
} from "./types.ts";
import { WATCH_CAPACITY, type Watch } from "./view.ts";
export type PublishedConversationView = ConversationView & {
commit: { events: ViewEvent[] };
};
export interface PicoConversationService {
readonly view: ReplicatedState<PublishedConversationView>;
send(input: SendInput, ctx: Context): Promise<Id>;
write(entry: NewEntry, ctx: Context): Promise<Id>;
inputAbort(id: Id, ctx: Context): Promise<"aborted" | "already_placed" | "not_found">;
configSet(patch: JsonObject, ctx: Context): Promise<void>;
abort(ctx: Context): Promise<void>;
reset(handoff: string | null, ctx: Context): Promise<void>;
collapse(instructions: string | null, ctx: Context): Promise<Id>;
fork(at: Id | "start", spec: Omit<ConversationSpec, "parent">, ctx: Context): Promise<Id>;
entries(scan: EntryScan, ctx: Context): Promise<Entry[]>;
}
/** Local registration/control plane and remote keyed conversation contract used by Chord facets. */
export const PicoHarnessService = defineService<Harness>("pi.harness", { local: true });
export const PicoConversationService = defineService<PicoConversationService>("pi.conversation");
export function createPicoConversationService<Cfg extends object>(
harness: Pick<Harness, "abortInput" | "entries">,
conversation: ConversationHandle<Cfg>,
view: ReplicatedState<PublishedConversationView>,
): PicoConversationService {
return {
view,
send: (input, ctx) => conversation.send(input, ctx).then((handle) => handle.id),
write: (entry, ctx) => conversation.write(entry, ctx),
inputAbort: (id, ctx) => harness.abortInput(id, ctx, conversation.id),
configSet: (patch, ctx) => conversation.config.set(patch as never, ctx),
abort: (ctx) => conversation.abort(ctx),
reset: (handoff, ctx) => conversation.reset(handoff ?? undefined, ctx),
collapse: (instructions, ctx) => conversation.collapse(instructions ?? undefined, ctx),
fork: (at, spec, ctx) => conversation.fork(at, spec, ctx).then((child) => child.id),
entries: (scan, ctx) => harness.entries({ ...scan, conversationId: conversation.id }, ctx),
};
}
export interface ChordViewBridge {
readonly view: MutableReplicatedState<PublishedConversationView>;
readonly closed: boolean;
close(): void;
}
export interface ChordViewBridgeOptions {
capacity?: number;
/** Called after the raw watch is closed. The owner should close and respawn its keyed service instance. */
onFailure?: (error: Error) => void;
}
/**
* Adapt Pico's commit-granular envelopes to one Chord publication per commit.
* The raw listener only enqueues. Applying ops and publishing happen together,
* off the Session line, with no await between them.
*/
export async function attachChordView(
conversation: Pick<ConversationHandle, "watch">,
createState: (initial: PublishedConversationView) => MutableReplicatedState<PublishedConversationView>,
ctx: Context,
opts: ChordViewBridgeOptions = {},
): Promise<ChordViewBridge> {
const watch = await conversation.watch(ctx);
const view = createState({ ...structuredClone(watch.view), commit: { events: [] } });
return bridgeWatch(watch, view, ctx, opts);
}
function bridgeWatch(
watch: Watch,
view: MutableReplicatedState<PublishedConversationView>,
ctx: Context,
opts: ChordViewBridgeOptions,
): ChordViewBridge {
const capacity = opts.capacity ?? WATCH_CAPACITY;
if (!Number.isSafeInteger(capacity) || capacity < 1)
throw new RangeError("Chord view queue capacity must be positive");
const queue: Envelope[] = [];
let scheduled = false;
let closed = false;
const fail = (cause: unknown) => {
if (closed) return;
closed = true;
queue.length = 0;
watch.stop();
try {
opts.onFailure?.(cause instanceof Error ? cause : new Error(String(cause)));
} catch {}
};
const drain = () => {
scheduled = false;
while (!closed && queue.length > 0) {
const envelope = queue.shift()!;
try {
applyTracked(view.state, envelope.ops);
view.state.commit.events = structuredClone(envelope.events);
view.publish(ctx);
} catch (error) {
fail(error);
}
}
};
watch.start((envelope) => {
if (closed) return;
if (queue.length >= capacity) {
fail(new Error(`Pico-to-Chord view queue exceeded ${capacity} envelopes`));
return;
}
queue.push(envelope);
if (scheduled) return;
scheduled = true;
queueMicrotask(drain);
});
return {
view,
get closed() {
return closed;
},
close() {
if (closed) return;
closed = true;
queue.length = 0;
watch.stop();
},
};
}
function applyTracked(root: PublishedConversationView, ops: readonly Op[]): void {
for (const op of ops) {
if (op[0] === "r") throw new Error("live Pico envelope unexpectedly replaced the view root");
const path = op[1];
if (op[0] === "p") {
const target = resolve(root, path);
if (!Array.isArray(target)) throw new Error(`Pico splice path is not an array: ${path.join(".")}`);
target.splice(op[2], op[3], ...structuredClone(op[4]));
continue;
}
const parent = resolve(root, path.slice(0, -1));
if (parent === null || typeof parent !== "object")
throw new Error(`Pico operation parent is not an object: ${path.join(".")}`);
const key = path[path.length - 1]!;
if (op[0] === "d") {
if (Array.isArray(parent)) parent.splice(key as number, 1);
else delete (parent as Record<PropertyKey, unknown>)[key];
continue;
}
const record = parent as Record<PropertyKey, unknown>;
if (op[0] === "s") record[key] = structuredClone(op[2]);
else if (op[0] === "a") record[key] = `${String(record[key])}${op[2]}`;
else record[key] = String(record[key]).slice(op[2]);
}
}
function resolve(root: unknown, path: Path): unknown {
let value = root;
for (const segment of path) {
if (value === null || typeof value !== "object") return undefined;
value = (value as Record<Seg, unknown>)[segment];
}
return value;
}

View file

@ -0,0 +1,105 @@
import type { Context } from "@earendil-works/chord";
import type { ContextEdit, ContextView, Entry, Id, RequestMessage, Storage, StoredMessage } from "./types.ts";
/**
* pico §1.3, verbatim:
* H = newest fork-visible entry at or before T with a head
* from = H ? H.head : transcript start
* range = fork-visible entries from `from` through T
* edits = per target, newest edit in range wins
* entries = H ? [H, ...range without any head entries] : range
* model = concat(entries.map(e => edits[e.id] ? apply : e.model)), then reorder tool results
*
* Display-only entries (aborted/error assistants, pi.usage, model-less plugin entries) have no
* `model` and contribute nothing. Nothing here inspects an error string.
*/
export async function deriveContext(
storage: Storage,
conversationId: Id,
at: Id | undefined,
ctx: Context,
): Promise<ContextView> {
const [head] = await storage.scanEntries(
{ conversationId, withHead: true, ...(at === undefined ? {} : { before: at + 1 }), limit: 1 },
ctx,
);
const from = head?.head;
// Walk newest-first until we pass `from`.
const range: Entry[] = [];
let before = at === undefined ? undefined : at + 1;
for (;;) {
const page = await storage.scanEntries(
{ conversationId, ...(before === undefined ? {} : { before }), limit: 256 },
ctx,
);
let done = page.length < 256;
for (const e of page) {
if (from !== undefined && e.id < from) {
done = true;
break;
}
range.push(e);
}
if (done) break;
before = page[page.length - 1]!.id;
}
range.reverse();
const edits = new Map<Id, ContextEdit>();
for (const e of range) for (const edit of e.edits ?? []) edits.set(edit.target, edit); // later wins by iteration order
const entries = head === undefined ? range : [head, ...range.filter((e) => e.head === undefined)];
const messages: RequestMessage[] = [];
for (const e of entries) {
const edit = edits.get(e.id);
if (edit?.action === "omit") continue;
if (edit?.action === "replace") {
messages.push(...(edit.messages ?? []));
continue;
}
if (e.model) messages.push(...e.model);
}
return { head, entries, messages: reorderToolResults(messages) };
}
/** Results are appended as tools finish; put them back in call order, synthesising a missing one after a fork cut. */
type StoredAssistant = Extract<StoredMessage, { role: "assistant" }>;
type StoredToolResult = Extract<StoredMessage, { role: "toolResult" }>;
type StoredToolCall = Extract<StoredAssistant["content"][number], { type: "toolCall" }>;
function reorderToolResults(messages: RequestMessage[]): RequestMessage[] {
const out: RequestMessage[] = [];
for (let i = 0; i < messages.length; i++) {
const m = messages[i]!;
out.push(m);
if (m.role !== "assistant") continue;
const calls = (m as StoredAssistant).content.filter((c): c is StoredToolCall => c.type === "toolCall");
if (calls.length === 0) continue;
const results = new Map<string, StoredToolResult>();
let j = i + 1;
while (j < messages.length && messages[j]!.role === "toolResult") {
const r = messages[j] as StoredToolResult;
results.set(r.toolCallId, r);
j++;
}
for (const call of calls) {
out.push(
results.get(call.id) ??
({
role: "toolResult",
toolCallId: call.id,
toolName: call.name,
content: [
{ type: "text", text: "Tool result unavailable: history ends before this call completed." },
],
isError: true,
details: { reason: "missing_after_fork" },
timestamp: (m as StoredAssistant).timestamp,
} as StoredToolResult),
);
}
i = j - 1;
}
return out;
}

View file

@ -0,0 +1,812 @@
import type { Context } from "@earendil-works/chord";
import { withAbortSignal } from "@earendil-works/chord/context";
import type { TSchema } from "typebox";
import { createHookRunners, type HookRegistration } from "./hooks.ts";
import { chooseThrough, collapse } from "./kinds/collapse.ts";
import { entries as builtinEntries } from "./kinds/entries.ts";
import { generation } from "./kinds/generation.ts";
import { job } from "./kinds/job.ts";
import { plugin } from "./kinds/plugin.ts";
import { postTools } from "./kinds/post-tools.ts";
import { tool } from "./kinds/tool.ts";
import { Scheduler } from "./scheduler.ts";
import { isCoreKind, Session, type TxImpl } from "./session.ts";
import { type SystemSection, systemSections } from "./system.ts";
import {
type AnyKind,
type AnyToolDeclaration,
type ConfigFacade,
type ConfigOfKinds,
type ContextView,
type Conversation,
type ConversationSpec,
type DisjointConfig,
type DocRef,
type Entry,
type EntryKind,
type EntryScan,
Forbidden,
type HooksOf,
type HostTx,
type Id,
type Input,
type Invoker,
type JsonObject,
type JsonValue,
type Models,
type Namespace,
type NamespaceDefaults,
type NamespaceRegistration,
type NewEntry,
type PluginHandler,
type ProcessHost,
type RewindableState,
type Runtime,
type SendInput,
type StickyState,
type Storage,
type Task,
type ToolDeclaration,
type UserInput,
} from "./types.ts";
import { ViewManager, type Watch } from "./view.ts";
// ---------------------------------------------------------------------------
// Harness
// ---------------------------------------------------------------------------
export interface HarnessOptions<Ks extends readonly AnyKind[] = []> {
models: Models;
tools?: AnyToolDeclaration[];
/** Ordinary kinds authored with `defineTask`. Their `config` merges onto `c.config`; keys must be disjoint. */
taskKinds?: Ks;
sections?: SystemSection[];
plugins?: { [name: string]: PluginHandler };
processHost?: ProcessHost;
/** Clock for durable kernel timestamps and retry scheduling. */
now?: () => number;
root?: { rewindable?: Partial<RewindableState>; sticky?: Partial<StickyState> };
/** Errors from listeners, hooks, watches, and the scheduler. Never delivered as commit failures. */
onReport?: (error: unknown) => void;
}
export type BuiltinKinds = readonly [
typeof generation,
typeof tool,
typeof postTools,
typeof collapse,
typeof job,
typeof plugin,
];
export type ConfigFor<Ks extends readonly AnyKind[]> = ConfigOfKinds<[...BuiltinKinds, ...Ks]>;
export interface ConversationHandle<Cfg extends object = ConfigFor<[]>> {
readonly id: Id;
readonly config: ConfigFacade<Cfg>;
send(input: SendInput, ctx: Context): Promise<InputHandle>;
write(entry: NewEntry, ctx: Context): Promise<Id>;
commit<T>(fn: (tx: HostTx, ctx: Context) => T | Promise<T>, ctx: Context): Promise<T>;
rewindable(ctx: Context): Promise<RewindableState>;
sticky(ctx: Context): Promise<StickyState>;
context(ctx: Context): Promise<ContextView>;
fork(at: Id | "start", spec: Omit<ConversationSpec, "parent">, ctx: Context): Promise<ConversationHandle<Cfg>>;
collapse(instructions: string | undefined, ctx: Context): Promise<Id>;
reset(handoff: string | undefined, ctx: Context): Promise<void>;
abort(ctx: Context): Promise<void>;
waitForIdle(ctx: Context): Promise<void>;
hooks<T extends JsonObject, K extends AnyKind>(
namespace: Namespace<T>,
kind: K,
handlers: Partial<HooksOf<K>>,
opts?: { subtree?: boolean },
): () => void;
watch(ctx: Context): Promise<Watch>;
}
export interface InputHandle {
readonly id: Id;
result(ctx: Context): Promise<Input | undefined>;
wait(ctx: Context): Promise<Input>;
abort(ctx: Context): Promise<"aborted" | "already_placed" | "not_found">;
}
const KERNEL: Invoker = { type: "kernel" };
export class Harness<Ks extends readonly AnyKind[] = []> {
private readonly session: Session;
private readonly scheduler: Scheduler;
private readonly views: ViewManager;
private readonly kinds: Map<string, AnyKind>;
private readonly namespaces = new Map<string, NamespaceRegistration>();
private readonly tools = new Map<string, AnyToolDeclaration>();
private readonly sectionMap = new Map<string, SystemSection>();
private readonly entryKinds = new Map<string, EntryKind>();
private readonly revisions = { sections: 0, tools: 0 };
private readonly plugins: Map<string, PluginHandler>;
private readonly hookRegistrations: HookRegistration[] = [];
private readonly conversationListeners = new Set<(conversation: ConversationHandle<ConfigFor<Ks>>) => void>();
private readonly onReport: (error: unknown) => void;
private readonly now: () => number;
private readonly ctx: Context;
private readonly options: HarnessOptions<Ks>;
private resumed = false;
private suspended = false;
static async open<Ks extends readonly AnyKind[] = []>(
storage: Storage,
options: HarnessOptions<Ks> &
(DisjointConfig<[...BuiltinKinds, ...Ks]> extends true
? unknown
: { readonly __error: "config keys collide across kinds" }),
ctx: Context,
): Promise<Harness<Ks>> {
const h = new Harness<Ks>(storage, options, ctx);
await h.init();
return h;
}
private constructor(storage: Storage, options: HarnessOptions<Ks>, ctx: Context) {
this.ctx = ctx;
this.options = options;
this.now = options.now ?? Date.now;
this.onReport = (error) => {
try {
options.onReport?.(error);
} catch {}
};
// Fixed core, installed internally. User kinds may not replace or shadow a built-in.
const builtins: AnyKind[] = [generation, tool, postTools, collapse, job, plugin];
this.kinds = new Map(builtins.map((k) => [k.name, k]));
for (const k of options.taskKinds ?? []) {
if (k.name.startsWith("pi."))
throw new Error(`task kind "${k.name}": names beginning with "pi." are reserved`);
if (this.kinds.has(k.name)) throw new Error(`task kind "${k.name}" registered twice`);
this.kinds.set(k.name, k);
}
for (const t of options.tools ?? []) this.registerTool(t, "open");
for (const s of Object.values(systemSections)) this.sectionMap.set(s.key, s as SystemSection);
for (const s of options.sections ?? []) this.registerSection(s, "open");
for (const e of Object.values(builtinEntries)) this.entryKinds.set(e.kind, e as EntryKind);
this.plugins = new Map(Object.entries(options.plugins ?? {}));
this.session = new Session(storage, this.kinds, this.namespaces, this.now); // throws if the Storage already has an owner; validates config disjointness
this.session.onReport = this.onReport;
this.views = new ViewManager(this.session, this.onReport);
this.session.lineListeners.add((result) => this.views.update(result));
this.session.listeners.add((result) => {
this.views.deliver();
for (const conversation of result.changes.conversations) this.notifyConversation(conversation);
});
const self = this;
const hooksFor = createHookRunners(
() => this.hookRegistrations,
(c) => this.session.index.ancestors(c),
this.onReport,
);
this.scheduler = new Scheduler({
session: this.session,
kinds: this.kinds,
onReport: this.onReport,
ctx,
runtime(task, invoker, _ictx): Runtime {
return {
taskId: task.id,
conversationId: task.conversationId,
kind: invoker.kind,
commit: (fn, c) => {
const owns = self.session.liveTasks.get(task.id)?.owns ?? [];
const docs = owns.flatMap((id): DocRef[] => [
{ doc: "rewindable", conversationId: id },
{ doc: "sticky", conversationId: id },
]);
return self.session
.commit(invoker, (tx, lineCtx) => fn(tx, self.session.liveTasks.get(task.id)!, lineCtx), c, {
docs,
})
.then((r) => r.value);
},
hooks: hooksFor(invoker.kind, { taskId: task.id, conversationId: task.conversationId }),
models: options.models,
tools: self.tools,
registries: {
sections: {
map: self.sectionMap,
get revision() {
return self.revisions.sections;
},
},
tools: {
map: self.tools,
get revision() {
return self.revisions.tools;
},
},
},
kinds: self.kinds,
processHost: options.processHost,
plugins: self.plugins,
now: self.now,
sleep: (untilMs, c) =>
new Promise((resolve, reject) => {
const ms = Math.max(0, untilMs - Date.now());
const t = setTimeout(() => {
c.abortSignal?.removeEventListener("abort", onAbort);
resolve();
}, ms);
const onAbort = () => {
clearTimeout(t);
reject(c.abortSignal?.reason ?? new Error("aborted"));
};
c.abortSignal?.addEventListener("abort", onAbort, { once: true });
}),
waitForInput: (id, c) => self.scheduler.waitForInput(id, c),
waitForTask: (id, c) => self.scheduler.waitForTask(id, c),
abortTask: async (id, c) => {
await self.session.onLine(async (lineCtx) => {
self.assertInvocation(invoker);
const target = self.session.liveTasks.get(id) ?? (await self.session.storage.task(id, lineCtx));
if (target === undefined) throw new Error(`task ${id} not found`);
self.assertTaskConversationScope(task.id, target.conversationId);
}, c);
return self.scheduler.abortTask(id, c);
},
abortConversation: async (id, c) => {
await self.session.onLine(() => {
self.assertInvocation(invoker);
self.assertOwnedConversation(task.id, id);
}, c);
const h = await self.conversation(id, c);
if (h) await h.abort(c);
},
createOwnedConversation: (spec, c) =>
self.session
.commit(
KERNEL,
(tx) => {
self.assertInvocation(invoker);
return tx.createOwnedConversation(task.id, task.conversationId, spec);
},
c,
)
.then((result) => result.value),
sendOwned: (id, input, c) =>
self.session
.commit(
KERNEL,
(tx) => {
self.assertInvocation(invoker);
self.assertOwnedConversation(task.id, id);
return tx.send(id, input);
},
c,
{
docs: [
{ doc: "rewindable", conversationId: id },
{ doc: "sticky", conversationId: id },
],
},
)
.then((r) => r.value),
context: (c, at, cx) => self.session.commit(invoker, (tx) => tx.context(c, at), cx).then((r) => r.value),
newestEntry: (c, opts, cx) =>
self.session.commit(invoker, (tx) => tx.newestEntry(c, opts), cx).then((r) => r.value),
rewindable: (c, cx) =>
self.session
.commit(invoker, (tx) => tx.snapshot({ doc: "rewindable", conversationId: c }), cx, {
docs: [{ doc: "rewindable", conversationId: c }],
})
.then((r) => r.value),
sticky: (c, cx) =>
self.session
.commit(invoker, (tx) => tx.snapshot({ doc: "sticky", conversationId: c }), cx, {
docs: [{ doc: "sticky", conversationId: c }],
})
.then((r) => r.value),
rewindableAsOf: (c, at, cx) =>
self.session.commit(invoker, (tx) => tx.rewindableAsOf(c, at), cx).then((r) => r.value),
};
},
});
}
private notifyConversation(conversation: Conversation): void {
if (this.conversationListeners.size === 0) return;
const handle = this.handle(conversation);
for (const listener of [...this.conversationListeners]) {
try {
listener(handle);
} catch (error) {
this.onReport(error);
}
}
}
private assertInvocation(invoker: Extract<Invoker, { type: "task" }>): void {
if (!invoker.token.alive) throw new Forbidden("operation from a finished invocation");
const live = this.session.liveTasks.get(invoker.id);
if (live === undefined) throw new Forbidden("operation from a task that is not live");
if (invoker.mode === "run" && live.abort === true) throw new Forbidden("operation from a marked run invocation");
}
private assertTaskConversationScope(taskId: Id, conversationId: Id): void {
const task = this.session.liveTasks.get(taskId);
if (task?.conversationId === conversationId) return;
if (task?.owns.some((root) => this.session.index.subtree(root).has(conversationId))) return;
throw new Forbidden(`conversation ${conversationId} is outside task ${taskId}'s subtree`);
}
private assertOwnedConversation(taskId: Id, conversationId: Id): void {
const task = this.session.liveTasks.get(taskId);
if (task === undefined || !task.owns.some((root) => this.session.index.subtree(root).has(conversationId))) {
throw new Forbidden(`conversation ${conversationId} is not owned by task ${taskId}`);
}
}
private async init() {
const { session } = this;
const convs = await session.read((s, ctx) => s.conversations(ctx), this.ctx);
for (const c of convs) session.conversationRecords.set(c.id, c);
const live = await session.read((s, ctx) => s.scanTasks({ status: ["pending", "running"] }, ctx), this.ctx);
for (const task of live) session.liveTasks.set(task.id, task);
// Owner tasks that are terminal but still own existing conversations: needed for ancestry.
const owners = new Set(convs.map((c) => c.owner).filter((o): o is Id => o !== undefined));
for (const id of owners)
if (!session.liveTasks.has(id)) {
const t = await session.read((s, ctx) => s.task(id, ctx), this.ctx);
if (t) session.ownerTaskCache.set(id, t);
}
if (convs.length === 0) {
await session.commit(
KERNEL,
(tx) =>
tx.createConversation({ rewindable: this.options.root?.rewindable, sticky: this.options.root?.sticky }),
this.ctx,
);
}
}
resume() {
if (this.suspended) throw new Error("cannot resume a suspended harness; reopen storage with a new harness");
if (this.resumed) return;
this.resumed = true;
void this.reconcileOrphans()
.then(() => {
if (!this.suspended) this.scheduler.resume();
})
.catch((error) => this.onReport(error));
}
private async reconcileOrphans(): Promise<void> {
const orphaned = [...this.session.liveTasks.values()].filter((task) => !this.kinds.has(task.kind));
if (orphaned.length === 0) return;
const docs = [...new Set(orphaned.map((task) => task.conversationId))].map(
(conversationId): DocRef => ({ doc: "sticky", conversationId }),
);
await this.session.commit(
KERNEL,
(tx) => {
for (const task of orphaned) tx.setTask({ ...task, status: "terminal", outcome: { status: "orphaned" } });
},
this.ctx,
{ docs },
);
}
quiescent(): boolean {
return this.scheduler.quiescent();
}
hold(): () => void {
if (!this.scheduler.quiescent()) throw new Error("cannot hold a non-quiescent harness; suspend it instead");
return this.scheduler.hold();
}
/** Cancel and join in-process invocations, clear transient waits, then close without terminalizing tasks. */
async suspend(ctx: Context): Promise<void> {
if (this.suspended) return;
this.suspended = true;
await this.scheduler.joinAll();
const tools = [...this.session.liveTasks.values()].filter((task) => task.kind === "pi.tool");
if (tools.length > 0) {
const docs = [...new Set(tools.map((task) => task.conversationId))].map(
(conversationId): DocRef => ({ doc: "sticky", conversationId }),
);
await this.session.commit(
KERNEL,
(tx) => {
for (const task of tools) {
const index = (task.input as { index: number }).index;
const slot = tx.sticky(task.conversationId).turn.tools[index];
if (slot?.waitingOn !== undefined) delete slot.waitingOn;
}
},
ctx,
{ docs },
);
}
this.views.close();
await this.session.close(ctx);
}
// --- registries --------------------------------------------------------------
registerTaskKind<K extends AnyKind>(kind: K): () => void {
if (kind.name.startsWith("pi."))
throw new Error(`task kind "${kind.name}": names beginning with "pi." are reserved`);
if (this.kinds.has(kind.name)) throw new Error(`task kind "${kind.name}" already registered`);
this.session.defaults.register(kind);
this.kinds.set(kind.name, kind);
this.scheduler.kick();
return () => {
if (this.kinds.get(kind.name) !== kind) return;
this.kinds.delete(kind.name);
this.session.defaults.unregister(kind);
};
}
namespace<T extends JsonObject>(
id: string,
defaults: NamespaceDefaults<T>,
opts: { view?: (slice: Readonly<T>) => JsonValue } = {},
): Namespace<T> {
if (!/^[a-z][a-z0-9_.-]*$/i.test(id) || id.startsWith("pi.")) throw new Error(`invalid namespace "${id}"`);
if (this.namespaces.has(id)) throw new Error(`namespace "${id}" already registered`);
const stored = JSON.parse(JSON.stringify(defaults)) as {
rewindable?: JsonObject;
sticky?: JsonObject;
session?: JsonObject;
};
const routes = new Map<string, "rewindable" | "sticky" | "session">();
for (const doc of ["rewindable", "sticky", "session"] as const) {
for (const key of Object.keys(stored[doc] ?? {})) {
if (routes.has(key))
throw new Error(`namespace "${id}" key "${key}" is declared in more than one document`);
routes.set(key, doc);
}
}
let token!: Namespace<T>;
token = {
id,
unregister: () => {
if (this.namespaces.get(id)?.token !== token) return;
this.namespaces.delete(id);
for (let index = this.hookRegistrations.length - 1; index >= 0; index--) {
if ((this.hookRegistrations[index]!.namespace as object) === token)
this.hookRegistrations.splice(index, 1);
}
},
};
const registration: NamespaceRegistration = {
token,
defaults: {
rewindable: stored.rewindable ?? {},
sticky: stored.sticky ?? {},
session: stored.session ?? {},
},
routes,
project: opts.view === undefined ? undefined : (slice) => opts.view!(slice as Readonly<T>),
};
this.namespaces.set(id, registration);
return token;
}
/** Register a tool. A duplicate name rejects. Unregister removes only this exact declaration; idempotent. */
registerTool(tool: AnyToolDeclaration, at: "open" | "runtime" = "runtime"): () => void {
if (this.tools.has(tool.name)) throw new Error(`tool "${tool.name}" already registered`);
this.tools.set(tool.name, tool);
this.revisions.tools++;
void at;
return () => {
if (this.tools.get(tool.name) === tool) {
this.tools.delete(tool.name);
this.revisions.tools++;
}
};
}
registerSection(section: SystemSection, at: "open" | "runtime" = "runtime"): () => void {
if (this.sectionMap.has(section.key)) throw new Error(`section "${section.key}" already registered`);
this.sectionMap.set(section.key, section);
this.revisions.sections++;
void at;
return () => {
if (this.sectionMap.get(section.key) === section) {
this.sectionMap.delete(section.key);
this.revisions.sections++;
}
};
}
registerEntryKind(kind: EntryKind): () => void {
if (kind.kind.startsWith("pi."))
throw new Error(`entry kind "${kind.kind}": names beginning with "pi." are reserved`);
if (this.entryKinds.has(kind.kind)) throw new Error(`entry kind "${kind.kind}" already registered`);
this.entryKinds.set(kind.kind, kind);
return () => {
if (this.entryKinds.get(kind.kind) === kind) this.entryKinds.delete(kind.kind);
};
}
/** Register namespace-bound handlers for one kind's hook points, harness-wide. Both tokens must be current. */
hooks<T extends JsonObject, K extends AnyKind>(
namespace: Namespace<T>,
kind: K,
handlers: Partial<HooksOf<K>>,
_opts?: Readonly<Record<string, never>>,
): () => void {
return this.addHooks({ namespace: this.checkNamespace(namespace), kind: this.checkKind(kind), handlers });
}
private checkKind(kind: AnyKind): AnyKind {
if (this.kinds.get(kind.name) !== kind) throw new Error(`kind "${kind.name}" is not the registered token`);
return kind;
}
private checkNamespace<T extends JsonObject>(namespace: Namespace<T>): Namespace<JsonObject> {
if (this.namespaces.get(namespace.id)?.token !== namespace)
throw new Forbidden(`namespace "${namespace.id}" is stale`);
return namespace as unknown as Namespace<JsonObject>;
}
private addHooks(reg: HookRegistration): () => void {
this.hookRegistrations.push(reg);
let done = false;
return () => {
if (done) return;
done = true;
const index = this.hookRegistrations.indexOf(reg);
if (index >= 0) this.hookRegistrations.splice(index, 1);
};
}
// --- conversations -------------------------------------------------------------
async root(ctx: Context): Promise<ConversationHandle<ConfigFor<Ks>>> {
return this.conversation(1, ctx).then((c) => c!);
}
onConversation(listener: (conversation: ConversationHandle<ConfigFor<Ks>>) => void): () => void {
this.conversationListeners.add(listener);
for (const conversation of this.session.conversationRecords.values()) {
try {
listener(this.handle(conversation));
} catch (error) {
this.onReport(error);
}
}
return () => this.conversationListeners.delete(listener);
}
async conversation(id: Id, ctx: Context): Promise<ConversationHandle<ConfigFor<Ks>> | undefined> {
const c = await this.session.read((s, lineCtx) => s.conversation(id, lineCtx), ctx);
return c === undefined ? undefined : this.handle(c);
}
async createConversation(
spec: ConversationSpec & { input?: UserInput },
ctx: Context,
): Promise<ConversationHandle<ConfigFor<Ks>>> {
const { input, ...rest } = spec;
const id = await this.session
.commit(
KERNEL,
async (tx) => {
const id = tx.createConversation(rest);
if (input !== undefined) await tx.send(id, { content: input });
return id;
},
ctx,
)
.then((r) => r.value);
return (await this.conversation(id, ctx))!;
}
entries(scan: EntryScan, ctx: Context): Promise<Entry[]> {
return this.session.read((s, lineCtx) => s.scanEntries(scan, lineCtx), ctx);
}
getTask(id: Id, ctx: Context): Promise<Task | undefined> {
return this.session.read((s, lineCtx) => s.task(id, lineCtx), ctx);
}
async abortInput(id: Id, ctx: Context, conversationId?: Id): Promise<"aborted" | "already_placed" | "not_found"> {
if (conversationId !== undefined) {
const input = await this.session.read((storage, lineCtx) => storage.input(id, lineCtx), ctx);
if (input !== undefined && input.conversationId !== conversationId)
throw new Forbidden(`input ${id} is outside conversation ${conversationId}`);
}
return this.inputHandle(id).abort(ctx);
}
abortTask(id: Id, ctx: Context): Promise<"marked" | "terminal"> {
return this.scheduler.abortTask(id, ctx);
}
/** Durably mark a task for abort without signalling its invocation; the scheduler aborts it when it next drains. */
markTask(id: Id, ctx: Context): Promise<"marked" | "terminal"> {
return this.session
.commit(
KERNEL,
async (tx) => {
const t = await tx.task(id);
if (t === undefined) throw new Error(`task ${id} not found`);
if (t.status === "terminal") return "terminal" as const;
tx.markTask(id);
return "marked" as const;
},
ctx,
)
.then((r) => r.value);
}
waitForIdle(ctx: Context): Promise<void> {
return this.scheduler.waitForIdle(undefined, ctx);
}
waitForTask(id: Id, ctx: Context): Promise<Task> {
return this.scheduler.waitForTask(id, ctx);
}
/** Signals every invocation, waits for them, closes storage. Writes nothing. */
async close(ctx: Context) {
await this.suspend(ctx);
}
private handle(c: Conversation): ConversationHandle<ConfigFor<Ks>> {
const self = this;
const docs: DocRef[] = [
{ doc: "rewindable", conversationId: c.id },
{ doc: "sticky", conversationId: c.id },
];
const hostInvoker: Invoker = { type: "host", conversationId: c.id };
const kernelInvoker: Invoker = { type: "kernel", conversationId: c.id };
const host = <T>(fn: (tx: HostTx, ctx: Context) => T | Promise<T>, ctx: Context) =>
self.session.commit(hostInvoker, (tx, lineCtx) => fn(tx, lineCtx), ctx, { docs }).then((r) => r.value);
const kernel = <T>(fn: (tx: TxImpl, ctx: Context) => T | Promise<T>, ctx: Context) =>
self.session.commit(kernelInvoker, fn, ctx, { docs }).then((r) => r.value);
const config: ConfigFacade<ConfigFor<Ks>> = {
get: (ctx) =>
host((tx) => {
const cfg = tx.config(c.id);
const out: { [k: string]: JsonValue | undefined } = {};
for (const key of self.session.defaults.route.keys()) out[key] = cfg.get(key);
return out as ConfigFor<Ks>;
}, ctx),
set: (patch, ctx) =>
host((tx) => {
const cfg = tx.config(c.id);
for (const [key, value] of Object.entries(patch)) {
if (value === undefined) throw new Error(`config.set(${key}): use config.reset()`);
cfg.set(key, value as JsonValue);
}
}, ctx),
reset: (keys, ctx) =>
host((tx) => {
const cfg = tx.config(c.id);
for (const key of keys) cfg.reset(String(key));
}, ctx),
};
return {
id: c.id,
config,
async send(input, ctx) {
const id = await kernel((tx) => tx.send(c.id, input), ctx);
return self.inputHandle(id);
},
write: (entry, ctx) => host((tx) => tx.write(c.id, entry), ctx),
commit: host,
rewindable: (ctx) => host((tx) => tx.snapshot({ doc: "rewindable", conversationId: c.id }), ctx),
sticky: (ctx) => host((tx) => tx.snapshot({ doc: "sticky", conversationId: c.id }), ctx),
context: (ctx) => host((tx) => tx.context(c.id), ctx),
async fork(at, spec, ctx) {
const id = await self.session.fork(c.id, at, spec, ctx);
return (await self.conversation(id, ctx))!;
},
collapse: (instructions, ctx) =>
kernel(async (tx) => {
const { entries } = await tx.context(c.id);
const state = tx.rewindable(c.id);
const through = chooseThrough(entries, state.keepRecent);
if (through === undefined) throw new Error("nothing to collapse");
return tx.createTask({
kind: "pi.collapse",
conversationId: c.id,
background: true,
input: { reason: "manual", through, ...(instructions === undefined ? {} : { instructions }) },
});
}, ctx),
reset: async (handoff, ctx) => {
await kernel(
(tx) =>
tx.write(
c.id,
handoff === undefined
? { kind: "pi.reset", head: "self" }
: {
kind: "pi.handoff",
head: "self",
model: [{ role: "user", content: handoff, timestamp: self.now() }],
},
),
ctx,
);
},
async abort(ctx) {
const owned: Id[] = [];
const marked = await kernel(async (tx) => {
const s = tx.sticky(c.id);
const withdrawn = s.inbox.filter((q) => q.mode !== "write").map((q) => q.id);
for (let i = s.inbox.length - 1; i >= 0; i--) if (s.inbox[i]!.mode !== "write") s.inbox.splice(i, 1);
await tx.resolveInputs(withdrawn, { status: "unanswered", reason: "aborted" });
for (const input of withdrawn) tx.emit({ type: "input.aborted", input });
const ids: Id[] = [];
for (const t of [...self.session.liveTasks.values()]) {
if (t.conversationId !== c.id || t.background) continue;
for (const o of t.owns) owned.push(o);
if (t.abort !== true) tx.markTask(t.id);
ids.push(t.id);
}
return ids;
}, ctx);
for (const id of marked) await self.scheduler.abortTask(id, ctx);
await self.scheduler.waitForIdle(c.id, ctx);
await Promise.all(owned.map((o) => self.scheduler.waitForIdle(o, ctx)));
},
waitForIdle: (ctx) => self.scheduler.waitForIdle(c.id, ctx),
hooks: (namespace, kind, handlers, opts) =>
self.addHooks({
namespace: self.checkNamespace(namespace),
kind: self.checkKind(kind),
handlers,
conversationId: c.id,
subtree: opts?.subtree,
}),
watch: (ctx) => self.watch(c, ctx),
};
}
/** Capture and subscribe in one line operation. */
private async watch(conversation: Conversation, ctx: Context): Promise<Watch> {
return this.session
.commit(
{ type: "host", conversationId: conversation.id },
async (tx) => {
const entries = await captureActiveTranscript((scan) => tx.scanEntries(scan), conversation.id);
return this.views.watch(conversation, entries);
},
ctx,
{
docs: [
{ doc: "rewindable", conversationId: conversation.id },
{ doc: "sticky", conversationId: conversation.id },
],
},
)
.then((result) => result.value);
}
private inputHandle(id: Id): InputHandle {
return {
id,
result: (ctx) => this.session.read((s, lineCtx) => s.input(id, lineCtx), ctx),
wait: (ctx) => this.scheduler.waitForInput(id, ctx),
abort: (ctx) =>
this.session.commit(KERNEL, (tx) => tx.withdrawInput(id), ctx, { docs: [] }).then((r) => r.value),
};
}
}
/**
* §9: H = newest fork-visible entry with a head. Active transcript = H plus every fork-visible entry
* with id ≥ H.head, chronological, nothing dropped inside the range; the whole transcript if no head.
*/
export async function captureActiveTranscript(
scan: (s: EntryScan) => Promise<Entry[]>,
conversationId: Id,
): Promise<Entry[]> {
const [h] = await scan({ conversationId, withHead: true, limit: 1 });
const from = h?.head;
const out: Entry[] = [];
let before: Id | undefined;
for (;;) {
const page = await scan({ conversationId, ...(before === undefined ? {} : { before }), limit: 256 });
let done = page.length < 256;
for (const e of page) {
if (from !== undefined && e.id < from) {
done = true;
break;
}
out.push(e);
}
if (done) break;
before = page[page.length - 1]!.id;
}
return out.reverse();
}
export { withAbortSignal, isCoreKind };
export type { ConversationView, Envelope, ViewEvent } from "./types.ts";
export type { Watch } from "./view.ts";
export { applyEnvelope, WATCH_CAPACITY } from "./view.ts";
/** The built-in kinds, as typed witnesses for `api.task`, `waitForTask`, and `hooks(kind, …)`. */
export const kinds = { generation, tool, postTools, collapse, job, plugin } as const;
export const entries = builtinEntries;
export type { ToolDeclaration, TSchema };

View file

@ -0,0 +1,47 @@
import type { AnyKind, HookInfo, HookRunner, Id, Namespace } from "./types.ts";
export interface HookRegistration {
namespace: Namespace;
kind: AnyKind;
handlers: object;
conversationId?: Id;
subtree?: boolean;
}
export function createHookRunners(
registrations: () => readonly HookRegistration[],
ancestors: (conversationId: Id) => readonly Id[],
onReport: (error: unknown) => void,
) {
return <H extends object>(kind: AnyKind, info: Omit<HookInfo, "kind">): HookRunner<H> => {
const handlers = () =>
registrations()
.filter((registration) => registration.kind === kind)
.filter(
(registration) =>
registration.conversationId === undefined ||
registration.conversationId === info.conversationId ||
(registration.subtree === true &&
ancestors(info.conversationId).includes(registration.conversationId)),
)
.map((registration) => ({
handlers: registration.handlers as Partial<H>,
namespace: registration.namespace,
api: { ...info, kind: kind.name },
}));
return {
handlers,
async each(ctx, fn, onValue) {
for (const binding of handlers()) {
try {
const v = await fn(binding.handlers, binding.api);
if (v !== undefined && onValue?.(v as never) === true) return;
} catch (error) {
if (ctx.abortSignal?.aborted) throw error;
onReport(error);
}
}
},
};
};
}

View file

@ -0,0 +1,153 @@
/**
* Experimental Pico3 kernel API.
*
* This subpath is intentionally separate from the package root while the kernel
* and its Chord integration are being validated.
*/
export { bashTool } from "./bash.ts";
export { Bounded } from "./bounded.ts";
export type {
ChordViewBridge,
ChordViewBridgeOptions,
PublishedConversationView,
} from "./chord.ts";
export {
attachChordView,
createPicoConversationService,
PicoConversationService,
PicoHarnessService,
} from "./chord.ts";
export type {
BuiltinKinds,
ConfigFor,
ConversationHandle,
HarnessOptions,
InputHandle,
Watch,
} from "./harness.ts";
export {
applyEnvelope,
captureActiveTranscript,
entries,
Harness,
isCoreKind,
kinds,
WATCH_CAPACITY,
withAbortSignal,
} from "./harness.ts";
export { JsonlStorage } from "./jsonl.ts";
export { MemoryStorage } from "./memory.ts";
export type { SystemSection } from "./system.ts";
export { defineSystemSection, systemSections } from "./system.ts";
export type {
AbortClosure,
AnyKind,
AnyToolDeclaration,
AssistantMessage,
BeforeToolApi,
Checkpoint,
CheckpointAt,
Closure,
Completion,
ConfigFacade,
ConfigOf,
ConfigOfKinds,
ConfigShape,
ContextEdit,
ContextView,
Conversation,
ConversationSpec,
ConversationView,
DeferredHandle,
DisjointConfig,
DocRef,
Entry,
EntryInput,
EntryKind,
EntryRef,
EntryScan,
Envelope,
GenerationStatus,
HookApi,
HookBinding,
HookInfo,
HookResult,
HookRunner,
HooksOf,
HostTx,
Id,
Input,
InputOf,
JsonObject,
JsonPrimitive,
JsonValue,
Kind,
KindConfig,
KindTypes,
Message,
Model,
ModelRef,
Models,
Namespace,
NamespaceDefaults,
NewEntry,
Outcome,
OwnedConversation,
OwnedConversationSpec,
PhaseHandler,
PluginHandler,
ProcessHost,
ProcessSpec,
ProcessStatus,
QueuedInput,
RequestMessage,
RequestOptions,
RetryPolicy,
RewindableState,
Runtime,
SendInput,
Seq,
SessionState,
SlotOf,
Step,
StickyState,
Storage,
Stored,
StoredMessage,
SystemMessage,
Task,
TaskOf,
TaskRef,
TaskScan,
TaskTx,
ThinkingLevel,
ToolApi,
ToolCall,
ToolControl,
ToolDeclaration,
ToolDiagnostic,
ToolResult,
ToolResultMessage,
ToolSlot,
TurnState,
TurnView,
UserInput,
UserMessage,
ViewEvent,
Write,
} from "./types.ts";
export {
Closed,
CollapseInProgress,
ConversationBusy,
defineEntry,
defineTask,
Faulted,
Forbidden,
GenerationInProgress,
memoOnce,
ReadAfterWrite,
TaskContractFault,
toStored,
} from "./types.ts";

View file

@ -0,0 +1,339 @@
import {
closeSync,
existsSync,
fsyncSync,
ftruncateSync,
mkdirSync,
openSync,
readdirSync,
readFileSync,
renameSync,
statSync,
unlinkSync,
writeSync,
} from "node:fs";
import { join } from "node:path";
import type { Context } from "@earendil-works/chord";
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import { isBase, type Op } from "@earendil-works/chord/delta";
import { MemoryStorage } from "./memory.ts";
import type { DocRef, Id, Seq, Storage, Write } from "./types.ts";
/** One line of any file. `refs` (main only) lists the sidecar files this commit also wrote. */
interface Record_ {
seq: Seq;
maxId: Id;
writes: Write[];
refs?: string[];
}
/**
* Files in a session directory:
* main.jsonl conversations, entries, inputs, rewindable/session doc ops, each task's
* CREATE and TERMINAL records, and one marker per commit. Append-only, never rewritten.
* sticky-<conversation>.jsonl sticky doc ops; rewritten from its last base by `truncate` (on the Session line)
* task-<id>.jsonl a live task's intermediate patches (running, checkpoints); unlinked after its
* terminal record is published in main
*
* Publication (§10): one commit `Seq`; sidecar records are appended first, then exactly one main
* record, last, listing the sidecar refs it expects — the publication point. Replay applies a sidecar
* record only when main has the marker for that seq naming that file; unconfirmed sidecar tails are
* ignored. Every record carries the committed-ID high-water. Bytes after the last newline of any file
* are a torn write and are truncated before the file is opened for append. There is no compaction.
*
* With `fsync: false`, recovery covers process termination while the OS and filesystem remain alive.
* It does not promise that an acknowledged commit survives power loss, kernel/host failure,
* storage-cache loss, or a filesystem that loses/reorders completed writes. An acknowledged tail may
* roll back, or a surviving main marker may reference a missing sidecar and make open fail rather
* than expose partial state. A lost pre-effect checkpoint can cause an external effect to be attempted
* again; external idempotency remains required.
*
* With `fsync: true`, each sidecar is flushed before the main publication marker is flushed. This
* strengthens file-data durability and ordering across machine failure. It is not a complete database
* guarantee: newly created files, renames, and unlinks are not followed by a parent-directory fsync,
* and storage hardware/filesystems may provide weaker guarantees.
*
* One process owns a directory at a time; a second process is unsupported.
*/
export class JsonlStorage extends MemoryStorage implements Storage {
private mainFd: number;
private readonly sidecars = new Map<Id, number>(); // sticky docs, by conversation
private readonly taskSidecars = new Map<Id, number>();
private readonly dir: string;
readonly fsync: boolean;
private closedOnce = false;
private constructor(dir: string, fsync: boolean) {
super();
this.dir = dir;
this.fsync = fsync;
this.mainFd = openSync(join(dir, "main.jsonl"), "a");
}
static async open(dir: string, opts: { fsync?: boolean } = {}): Promise<JsonlStorage> {
mkdirSync(dir, { recursive: true });
for (const f of readdirSync(dir)) if (f.endsWith(".jsonl")) truncateTornTail(join(dir, f));
const storage = new JsonlStorage(dir, opts.fsync ?? true);
try {
await storage.replay();
return storage;
} catch (error) {
await storage.close();
throw error;
}
}
/** For tests and tooling: sizes of every file in the directory. */
sizes(): { [file: string]: number } {
const out: { [file: string]: number } = {};
for (const f of readdirSync(this.dir)) out[f] = statSync(join(this.dir, f)).size;
return out;
}
private async replay() {
const read = (file: string): { record: Record_; end: number }[] => {
if (!existsSync(file)) return [];
const bytes = readFileSync(file);
const out: { record: Record_; end: number }[] = [];
let start = 0;
let last = 0;
for (let end = bytes.indexOf(0x0a, start); end >= 0; end = bytes.indexOf(0x0a, start)) {
const line = bytes.subarray(start, end).toString("utf8");
start = end + 1;
if (line === "") continue;
let record: Record_;
try {
record = JSON.parse(line) as Record_;
} catch (error) {
throw new Error(`${file}: malformed record: ${String(error)}`);
}
if (typeof record.seq !== "number" || typeof record.maxId !== "number" || !Array.isArray(record.writes))
throw new Error(`${file}: record lacks seq/maxId/writes`);
if (record.seq <= last) throw new Error(`${file}: sequence not increasing at ${record.seq}`);
last = record.seq;
out.push({ record, end: start });
}
return out;
};
let maxId = 0;
// Main is the commit log. Confirmed = (seq → refs) for every marker.
const main = read(join(this.dir, "main.jsonl")).map(({ record }) => record);
const confirmed = new Map<Seq, Set<string>>();
for (const record of main) confirmed.set(record.seq, new Set(record.refs ?? []));
// Merge every file's records by seq so batches apply in commit order.
const sidecarFiles = readdirSync(this.dir).filter(
(file) => file.startsWith("sticky-") || file.startsWith("task-"),
);
const bySeq = new Map<Seq, Write[]>();
const found = new Map<Seq, Set<string>>();
const retainedStickyBase = new Map<string, Seq>();
const retiredTasks = new Map<Id, Seq>();
for (const record of main) {
for (const write of record.writes) {
if (write.type === "task.patch" && write.patch.status === "terminal")
retiredTasks.set(write.patch.id, record.seq);
}
bySeq.set(record.seq, [...record.writes]);
maxId = Math.max(maxId, record.maxId);
}
for (const file of sidecarFiles) {
const path = join(this.dir, file);
const records = read(path);
const first = records[0]?.record;
if (
file.startsWith("sticky-") &&
first !== undefined &&
isBase(first.writes.flatMap((write) => (write.type === "doc" ? write.ops : [])))
) {
retainedStickyBase.set(file, first.seq);
}
let confirmedEnd = 0;
let sawUnconfirmed = false;
for (const { record, end } of records) {
if (!confirmed.get(record.seq)?.has(file)) {
sawUnconfirmed = true;
continue;
}
if (sawUnconfirmed)
throw new Error(`${path}: confirmed record follows an unconfirmed tail at ${record.seq}`);
confirmedEnd = end;
const files = found.get(record.seq) ?? new Set<string>();
files.add(file);
found.set(record.seq, files);
bySeq.get(record.seq)!.push(...record.writes);
maxId = Math.max(maxId, record.maxId);
}
if (sawUnconfirmed) truncateTo(path, confirmedEnd);
}
for (const record of main) {
for (const file of record.refs ?? []) {
if (found.get(record.seq)?.has(file)) continue;
const retainedFrom = retainedStickyBase.get(file);
if (retainedFrom !== undefined && record.seq < retainedFrom) continue;
if (file.startsWith("task-")) {
const id = Number(file.slice(5, -6));
const retiredAt = retiredTasks.get(id);
if (retiredAt !== undefined && record.seq < retiredAt) continue;
}
throw new Error(`${join(this.dir, file)}: missing record for published sequence ${record.seq}`);
}
}
for (const seq of [...bySeq.keys()].sort((a, b) => a - b)) {
this.seq = seq - 1;
await super.commit(bySeq.get(seq)!, BACKGROUND_CONTEXT);
}
this.setNextId(maxId + 1);
// Task sidecars belong to live tasks only; a leftover for a terminal (or unknown) task cannot resurrect anything.
for (const f of sidecarFiles) {
if (!f.startsWith("task-")) continue;
const id = Number(f.slice(5, -6));
const t = await super.task(id);
if (t === undefined || t.status === "terminal") unlinkSync(join(this.dir, f));
}
}
override async commit(writes: readonly Write[], ctx: Context): Promise<Seq> {
if (this.closedOnce) throw new Error("storage closed");
const seq = this.seq + 1;
const apply = this.stage(writes, seq); // validates the whole batch first; throws before any file is touched
const maxId = this.nextIdAfter(writes);
const main: Write[] = [];
const sticky = new Map<Id, Write[]>();
const tasks = new Map<Id, Write[]>();
const retired: Id[] = [];
for (const w of writes) {
if (w.type === "doc" && w.ref.doc === "sticky")
(sticky.get(w.ref.conversationId) ?? sticky.set(w.ref.conversationId, []).get(w.ref.conversationId)!).push(
w,
);
else if (w.type === "task.patch" && w.patch.status !== "terminal")
(tasks.get(w.patch.id) ?? tasks.set(w.patch.id, []).get(w.patch.id)!).push(w);
else {
main.push(w);
if (w.type === "task.patch" && w.patch.status === "terminal") retired.push(w.patch.id);
}
}
const refs: string[] = [];
for (const [id, ws] of tasks) {
this.append(this.taskSidecar(id), { seq, maxId, writes: ws });
refs.push(`task-${id}.jsonl`);
}
for (const [c, ws] of sticky) {
this.append(this.sidecar(c), { seq, maxId, writes: ws });
refs.push(`sticky-${c}.jsonl`);
}
this.append(this.mainFd, { seq, maxId, writes: main, ...(refs.length ? { refs } : {}) }); // the publication point
apply();
this.seq = seq;
for (const id of retired) this.retireTaskSidecar(id); // after the terminal record is durably published
void ctx;
return seq;
}
private nextIdAfter(writes: readonly Write[]): Id {
let max = this.nextIdValue - 1;
for (const w of writes) {
const id =
w.type === "conversation"
? w.conversation.id
: w.type === "entry"
? w.entry.id
: w.type === "task"
? w.task.id
: w.type === "input"
? w.input.id
: 0;
max = Math.max(max, id);
}
return max;
}
private append(fd: number, record: Record_) {
writeSync(fd, `${JSON.stringify(record)}\n`);
if (this.fsync) fsyncSync(fd);
}
private sidecar(conversationId: Id): number {
let fd = this.sidecars.get(conversationId);
if (fd === undefined) {
fd = openSync(join(this.dir, `sticky-${conversationId}.jsonl`), "a");
this.sidecars.set(conversationId, fd);
}
return fd;
}
private taskSidecar(id: Id): number {
let fd = this.taskSidecars.get(id);
if (fd === undefined) {
fd = openSync(join(this.dir, `task-${id}.jsonl`), "a");
this.taskSidecars.set(id, fd);
}
return fd;
}
private retireTaskSidecar(id: Id) {
const fd = this.taskSidecars.get(id);
if (fd !== undefined) closeSync(fd);
this.taskSidecars.delete(id);
try {
unlinkSync(join(this.dir, `task-${id}.jsonl`));
} catch {}
}
/** Rewrite the sticky sidecar from its last base: temp file, optional fsync, rename. Called on the Session line. */
override async truncate(ref: DocRef, _ctx?: Context): Promise<void> {
await super.truncate(ref);
if (ref.doc !== "sticky") return;
const file = join(this.dir, `sticky-${ref.conversationId}.jsonl`);
if (!existsSync(file)) return;
const lines = readFileSync(file, "utf8").split("\n").filter(Boolean);
let start = 0;
for (let i = lines.length - 1; i >= 0; i--)
if (
isBase((JSON.parse(lines[i]!) as Record_).writes.flatMap((w) => (w.type === "doc" ? w.ops : [])) as Op[])
) {
start = i;
break;
}
if (start === 0) return;
const tmp = `${file}.tmp`;
const fd = openSync(tmp, "w");
writeSync(
fd,
lines
.slice(start)
.map((l) => `${l}\n`)
.join(""),
);
if (this.fsync) fsyncSync(fd);
closeSync(fd);
const old = this.sidecars.get(ref.conversationId);
if (old !== undefined) closeSync(old);
renameSync(tmp, file);
this.sidecars.set(ref.conversationId, openSync(file, "a"));
}
override async close(_ctx?: Context) {
if (this.closedOnce) return;
this.closedOnce = true;
await super.close();
closeSync(this.mainFd);
for (const fd of this.sidecars.values()) closeSync(fd);
for (const fd of this.taskSidecars.values()) closeSync(fd);
}
}
function truncateTo(file: string, length: number): void {
const fd = openSync(file, "r+");
try {
ftruncateSync(fd, length);
} finally {
closeSync(fd);
}
}
/** Bytes after the last newline are a torn write: cut them so a later append cannot extend a corrupt line. */
function truncateTornTail(file: string) {
const size = statSync(file).size;
if (size === 0) return;
const buf = readFileSync(file);
const end = buf.lastIndexOf(0x0a);
if (end + 1 !== buf.length) truncateTo(file, end + 1);
}

View file

@ -0,0 +1,274 @@
import type { Context } from "@earendil-works/chord";
import type { AssistantMessage } from "@earendil-works/pi-ai";
import { estimateContextTokens } from "@earendil-works/pi-ai/utils/estimate";
import { effectiveTools } from "../system.ts";
import {
type Completion,
type CoreKind,
type CoreTx,
type Entry,
type HookInfo,
type HookResult,
type Id,
type ModelRef,
type RequestMessage,
type RetryPolicy,
type Runtime,
type Step,
type SystemMessage,
type Task,
type ThinkingLevel,
toStored,
} from "../types.ts";
import { retryDecision } from "./generation.ts";
const estimate = (messages: RequestMessage[]) =>
estimateContextTokens(
messages.filter((message): message is Exclude<RequestMessage, SystemMessage> => message.role !== "system"),
).tokens;
export type CollapseInput = { reason: "threshold" | "manual" | "overflow"; through: Id; instructions?: string };
type Base = {
expectedHead: Id | null;
instructions?: string;
model: ModelRef;
thinkingLevel: ThinkingLevel;
retry: RetryPolicy;
attempt: number;
};
export type CollapseCheckpoint =
| ({ phase: "summarizing" } & Base)
| ({ phase: "retrying"; untilMs: number; lastError: string } & Base)
| ({ phase: "prepared"; summary: string } & Base);
export type CollapseFailure = {
reason: "stale" | "declined" | "provider" | "retries_exhausted" | "no_model";
detail: string;
};
export interface CollapseHooks {
beforeCollapse(
reason: "threshold" | "manual" | "overflow",
through: Id,
entries: Entry[],
info: HookInfo,
ctx: Context,
): HookResult<{ decline: true } | { instructions?: string; summary?: string }>;
}
export const collapseConfig = { rewindable: { threshold: 0 as number, keepRecent: 20_000 as number } } as const;
type CollapseTask = Task<CollapseInput, CollapseCheckpoint>;
type CollapseRuntime = Runtime<CollapseHooks, CoreTx>;
type CollapseStep = Step<CollapseCheckpoint, { summary: Id }, CollapseFailure, CoreTx>;
const failed = (reason: CollapseFailure["reason"], detail: string): Completion<{ summary: Id }, CollapseFailure> => ({
status: "failed",
failure: { reason, detail },
});
const baseOf = (checkpoint: CollapseCheckpoint): Base => {
const { phase: _phase, ...rest } = checkpoint;
const {
untilMs: _untilMs,
lastError: _lastError,
summary: _summary,
...base
} = rest as Base & {
untilMs?: number;
lastError?: string;
summary?: string;
};
return base;
};
export const collapse: CoreKind<
CollapseInput,
CollapseCheckpoint,
{ summary: Id },
CollapseFailure,
null,
CollapseHooks,
typeof collapseConfig
> = {
name: "pi.collapse",
config: collapseConfig,
inflight: ["summarizing"],
async initial(task, runtime, ctx) {
const state = await runtime.rewindable(task.conversationId, ctx);
if (state.model === undefined) return { done: () => failed("no_model", "no model configured") };
const sticky = await runtime.sticky(task.conversationId, ctx);
const head = await runtime.newestEntry(task.conversationId, { withHead: true }, ctx);
const { entries } = await runtime.context(task.conversationId, task.input.through, ctx);
let decision: { decline: true } | { instructions?: string; summary?: string } = {};
await runtime.hooks.each(
ctx,
(hook, api) => hook.beforeCollapse?.(task.input.reason, task.input.through, entries, api, ctx),
(value) => {
if (value === undefined) return;
decision = value;
return true;
},
);
if ("decline" in decision) return { done: () => failed("declined", "declined by beforeCollapse") };
const choice: { instructions?: string; summary?: string } = decision;
const base: Base = {
expectedHead: head?.id ?? null,
...((choice.instructions ?? task.input.instructions)
? { instructions: choice.instructions ?? task.input.instructions }
: {}),
model: state.model,
thinkingLevel: state.thinkingLevel,
retry: sticky.retry,
attempt: 1,
};
if (choice.summary !== undefined) {
return {
next: async (tx, current) =>
(await headMoved(tx, current.conversationId, base))
? failed("stale", "head moved during beforeCollapse")
: { phase: "prepared", summary: choice.summary!, ...base },
};
}
return summarizeNow(task, base, runtime, ctx, "check head");
},
phases: {
async summarizing(task, runtime) {
return afterFailure(baseOf(task.checkpoint), null, "interrupted", runtime);
},
async retrying(task, runtime, ctx) {
await runtime.sleep(task.checkpoint.untilMs, ctx);
return summarizeNow(task, { ...baseOf(task.checkpoint), attempt: task.checkpoint.attempt + 1 }, runtime, ctx);
},
async prepared(task, runtime) {
const { summary } = task.checkpoint;
const base = baseOf(task.checkpoint);
return {
done: async (tx, current) => {
const { head, entries } = await tx.context(current.conversationId);
if ((head?.id ?? null) !== base.expectedHead) return failed("stale", "head moved during collapse");
const retained = entries.find((entry) => entry.id > task.input.through);
const id = tx.appendEntry(current.conversationId, {
kind: "pi.summary",
data: { through: task.input.through },
model: [{ role: "user", content: summary, timestamp: runtime.now() }],
head: retained?.id ?? "self",
});
return { status: "completed", result: { summary: id } };
},
};
},
},
async abort() {
return async () => null;
},
};
const headMoved = async (tx: Pick<CoreTx, "newestEntry">, conversationId: Id, base: Base) =>
((await tx.newestEntry(conversationId, { withHead: true }))?.id ?? null) !== base.expectedHead;
async function summarizeNow(
task: CollapseTask,
base: Base,
runtime: CollapseRuntime,
ctx: Context,
check?: "check head",
): Promise<CollapseStep> {
const stale = await runtime.commit(async (tx, current) => {
if (check && (await headMoved(tx, current.conversationId, base))) return true;
tx.checkpoint({ phase: "summarizing", ...base });
return false;
}, ctx);
if (stale) return { done: () => failed("stale", "head moved during beforeCollapse") };
const model = runtime.models.resolve(base.model);
if (model === undefined) return { done: () => failed("no_model", "model unavailable") };
const { messages } = await runtime.context(task.conversationId, task.input.through, ctx);
const tools = effectiveTools(messages);
const request: RequestMessage[] = [
...messages,
...(tools.length > 0
? [toStored({ role: "system", content: "", toolsRemoved: tools, timestamp: runtime.now() } as SystemMessage)]
: []),
{
role: "user",
content: `${base.instructions ?? "Summarize the conversation so far for continuation."}\n\nRespond with the summary only.`,
timestamp: runtime.now(),
},
];
let message: AssistantMessage | undefined;
try {
for await (const event of runtime.models.stream(
model,
{ messages: request, thinkingLevel: base.thinkingLevel },
ctx,
)) {
if (event.type === "done") {
message = event.message;
break;
}
if (event.type === "error") {
message = event.error;
break;
}
}
} catch (error) {
if (ctx.abortSignal?.aborted) throw error;
return afterFailure(base, null, String(error), runtime);
}
if (message === undefined) return afterFailure(base, null, "no message", runtime);
if (message.content.some((content) => content.type === "toolCall")) {
return afterFailure(base, { ...message, stopReason: "error" }, "summarizer returned tool calls", runtime);
}
if (message.stopReason === "error")
return afterFailure(base, message, message.errorMessage ?? "provider error", runtime);
const summary = message.content
.filter((content): content is { type: "text"; text: string } => content.type === "text")
.map((content) => content.text)
.join("");
return { next: { phase: "prepared", summary, ...base } };
}
function afterFailure(
base: Base,
message: AssistantMessage | null,
detail: string,
runtime: CollapseRuntime,
): CollapseStep {
const decision = retryDecision(base, message, runtime.now());
if (decision.kind === "fail") return { done: () => failed(decision.reason, detail) };
return {
next: (tx, current) => {
tx.emit({
type: "compaction.retrying",
taskId: current.id,
attempt: base.attempt,
retryAt: decision.untilMs,
error: detail,
});
return { phase: "retrying", ...base, untilMs: decision.untilMs, lastError: detail };
},
};
}
export function chooseThrough(entries: Entry[], keepRecent: number): Id | undefined {
const exchanges: { last: Id; tokens: number }[] = [];
let open: { last: Id; tokens: number } | undefined;
for (const entry of entries) {
const role = entry.model?.[0]?.role;
const tokens = estimate((entry.model ?? []) as RequestMessage[]);
if (role === "assistant") {
open = { last: entry.id, tokens };
exchanges.push(open);
} else if (role === "toolResult" && open) {
open.last = entry.id;
open.tokens += tokens;
} else {
open = undefined;
exchanges.push({ last: entry.id, tokens });
}
}
let retained = 0;
let index = exchanges.length - 1;
while (index >= 0 && retained + exchanges[index]!.tokens <= keepRecent) retained += exchanges[index--]!.tokens;
if (index < 0) return undefined;
if (index === exchanges.length - 1) return exchanges[index - 1]?.last;
return exchanges[index]!.last;
}

View file

@ -0,0 +1,39 @@
import type { AssistantMessage, ToolResultMessage, UserMessage } from "@earendil-works/pi-ai";
import type { Entry, EntryKind, Id, JsonValue, SystemMessage, ToolControl } from "../types.ts";
export type UserEntry = Entry & { kind: "pi.user"; model: [UserMessage]; data?: { continuation: true; from: Id } };
export type AssistantEntry = Entry & { kind: "pi.assistant"; model: [AssistantMessage]; data: { attempt: number } };
export type ToolResultEntry = Entry & {
kind: "pi.tool_result";
model: [ToolResultMessage];
data: {
details?: JsonValue;
diagnostics?: JsonValue;
control?: ToolControl;
truncated?: { bytes: number; lines: number };
};
};
export type SystemEntry = Entry & { kind: "pi.system"; model: [SystemMessage]; data: { baseline: boolean } };
export type NoticeEntry = Entry & { kind: "pi.notice"; model: [UserMessage] };
export type UsageEntry = Entry & {
kind: "pi.usage";
data: { attempt: number; usage?: AssistantMessage["usage"]; error: string };
};
export type SummaryEntry = Entry & { kind: "pi.summary"; model: [UserMessage]; data: { through: Id }; head: Id };
export type HandoffEntry = Entry & { kind: "pi.handoff"; model: [UserMessage]; head: Id };
export type ResetEntry = Entry & { kind: "pi.reset"; head: Id };
const coreEntry = <E extends Entry>(kind: string): EntryKind<E> =>
Object.freeze({ kind, is: (entry: Entry | undefined): entry is E => entry?.kind === kind });
export const entries = {
user: coreEntry<UserEntry>("pi.user"),
assistant: coreEntry<AssistantEntry>("pi.assistant"),
toolResult: coreEntry<ToolResultEntry>("pi.tool_result"),
system: coreEntry<SystemEntry>("pi.system"),
notice: coreEntry<NoticeEntry>("pi.notice"),
usage: coreEntry<UsageEntry>("pi.usage"),
summary: coreEntry<SummaryEntry>("pi.summary"),
handoff: coreEntry<HandoffEntry>("pi.handoff"),
reset: coreEntry<ResetEntry>("pi.reset"),
} as const;

View file

@ -0,0 +1,76 @@
import type {
AssistantMessage,
AssistantMessageFrame,
TextContent,
ThinkingContent,
ToolCall,
} from "@earendil-works/pi-ai";
/**
* Apply one encoded frame to the tracked output. Same switch as pi-ai's
* reduceAssistantMessageFrames (the test oracle); no buffering, no second copy.
*/
export function applyFrame(o: { message?: AssistantMessage }, frame: AssistantMessageFrame): void {
if (frame.type === "start") {
o.message = frame.partial; // the encoder's clone
return;
}
const m = o.message;
if (m === undefined) throw new Error(`${frame.type} before start`);
const c = m.content;
switch (frame.type) {
case "text_start":
c[frame.contentIndex] = { ...frame.content };
return;
case "text_delta":
(c[frame.contentIndex] as TextContent).text += frame.delta;
return;
case "text_end": {
const b = c[frame.contentIndex] as TextContent;
b.text = frame.content;
delete b.textSignature;
if (frame.textSignature !== undefined) b.textSignature = frame.textSignature;
return;
}
case "thinking_start":
c[frame.contentIndex] = { ...frame.content };
return;
case "thinking_delta":
(c[frame.contentIndex] as ThinkingContent).thinking += frame.delta;
return;
case "thinking_end": {
const b = c[frame.contentIndex] as ThinkingContent;
b.thinking = frame.content;
delete b.thinkingSignature;
delete b.redacted;
if (frame.thinkingSignature !== undefined) b.thinkingSignature = frame.thinkingSignature;
if (frame.redacted !== undefined) b.redacted = frame.redacted;
return;
}
case "toolcall_start":
c[frame.contentIndex] = { ...frame.toolCall };
return;
case "toolcall_checkpoint":
(c[frame.contentIndex] as ToolCall).arguments = safeParse(frame.json);
return;
case "toolcall_delta":
return; // arguments materialise at checkpoint/end; deltas only feed the encoder's own json buffer
case "toolcall_end": {
const b = c[frame.contentIndex] as ToolCall;
b.id = frame.id;
b.name = frame.name;
b.arguments = frame.arguments;
if (frame.thoughtSignature !== undefined) b.thoughtSignature = frame.thoughtSignature;
if (frame.namespace !== undefined) b.namespace = frame.namespace;
return;
}
}
}
function safeParse(json: string): ToolCall["arguments"] {
try {
return JSON.parse(json);
} catch {
return {};
}
}

View file

@ -0,0 +1,641 @@
import type { Context } from "@earendil-works/chord";
import type {
AssistantMessage,
AssistantMessageEvent,
AssistantMessageFrame,
DeferredHandle,
ToolCall,
} from "@earendil-works/pi-ai";
import { AssistantMessageFrameEncoder, isRetryableAssistantError } from "@earendil-works/pi-ai";
import { estimateContextTokens } from "@earendil-works/pi-ai/utils/estimate";
import { planManagedEntry, prepareDraft, type SystemInstructionsHooks, sameSnapshot, takeSnapshot } from "../system.ts";
import {
type Closure,
type Completion,
type CoreKind,
type CoreTx,
type HookInfo,
type HookResult,
type Id,
type ModelRef,
type RequestMessage,
type RetryPolicy,
type Runtime,
type Step,
type Stored,
type Task,
type ThinkingLevel,
toStored,
} from "../types.ts";
import { chooseThrough } from "./collapse.ts";
import { applyFrame } from "./frames.ts";
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
export type GenerationInput = { inputs: Id[] };
/** Captured once at `prepared`; carried unchanged except `attempt`. */
type Prep = {
cutoff: Id;
system: Id | null;
model: ModelRef;
thinkingLevel: ThinkingLevel;
tools: string[];
retry: RetryPolicy;
attempt: number;
};
export type GenerationCheckpoint =
| ({ phase: "prepared" } & Prep)
| ({ phase: "requesting" } & Prep)
| ({ phase: "retrying"; untilMs: number; lastError: string } & Prep)
| ({ phase: "deferred"; handle: Stored<DeferredHandle>; pollAt: number } & Prep);
export type GenerationResult = { assistant: Id; tools: Id[]; postTools?: Id; successor?: Id };
export type GenerationFailure = {
reason: "provider" | "overflow" | "retries_exhausted" | "no_model";
detail: string;
assistant?: Id;
};
/** Display-only assistant entries (provider error, aborted partial) carry the message here, never in `model`. */
export type DisplayAssistantData = { attempt: number; display: Stored<AssistantMessage>; reason: "error" | "aborted" };
/** Hook points this kind calls. Register with `h.hooks(namespace, kinds.generation, { … })`. */
export interface GenerationHooks extends SystemInstructionsHooks {
/** Before every request (also on retry and recovery). Chain. */
beforeRequest(
request: { messages: RequestMessage[] },
info: HookInfo & { cutoff: Id },
ctx: Context,
): HookResult<{ messages: RequestMessage[] }>;
/** Every terminal provider message, before classification. Observer. */
afterResponse(message: AssistantMessage, info: HookInfo & { attempt: number }, ctx: Context): void | Promise<void>;
/** On a final answer with nothing queued. First `{ continue }` wins and starts a continuation. */
onYield(answer: AssistantMessage, info: HookInfo, ctx: Context): HookResult<{ continue: string }>;
}
/** Configuration this kind reads, with defaults. `model` has none: absent → failed/no_model. */
export const generationConfig = {
rewindable: {
model: undefined as ModelRef | undefined,
thinkingLevel: "off" as ThinkingLevel,
selectedTools: [] as string[],
profile: "default" as string,
},
sticky: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 2000, maxAgentDelayMs: 60_000 } as RetryPolicy },
} as const;
type G = Task<GenerationInput, GenerationCheckpoint>;
type Rt = Runtime<GenerationHooks, CoreTx>;
type S = Step<GenerationCheckpoint, GenerationResult, GenerationFailure, CoreTx>;
type Out = Closure<GenerationResult, GenerationFailure, CoreTx>;
const fail = (
reason: GenerationFailure["reason"],
detail: string,
assistant?: Id,
): Completion<GenerationResult, GenerationFailure> => ({
status: "failed",
failure: { reason, detail, ...(assistant === undefined ? {} : { assistant }) },
});
/** Every terminal generation failure settles its group and admits queued triggers at the final boundary. */
const failStep = (task: G, reason: GenerationFailure["reason"], detail: string): S => ({
done: async (tx, current) => {
const head = await tx.newestEntry(current.conversationId, { withHead: true });
tx.emit({ type: "generation.failed", taskId: current.id, reason, detail });
await settleFailedTurn(tx, current, task.input.inputs, detail, head?.id);
return fail(reason, detail);
},
});
/** pi-ai drops system messages and aborted/error assistant messages before sending; estimate what is sent. */
const estimate = (messages: RequestMessage[]) =>
estimateContextTokens(
messages.filter(
(m) =>
m.role !== "system" &&
!(m.role === "assistant" && (m.stopReason === "aborted" || m.stopReason === "error")),
) as never,
).tokens;
const emptyUsage = (): AssistantMessage["usage"] => ({
input: 0,
output: 0,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 0,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
});
// ---------------------------------------------------------------------------
// The kind
// ---------------------------------------------------------------------------
export const generation: CoreKind<
GenerationInput,
GenerationCheckpoint,
GenerationResult,
GenerationFailure,
{ assistant?: Id },
GenerationHooks,
typeof generationConfig
> = {
name: "pi.generation",
turn: true,
config: generationConfig,
inflight: ["requesting"],
// Step 1 (§12.5): snapshot S on the line; systemInstructions off the line; then in one
// commit re-take S′, retry if it moved, else append the managed entry and checkpoint
// `prepared` with the cutoff. Runs once per generation; never on recovery.
async initial(task, rt, ctx) {
const { snapshot, canonical, seed } = await rt.commit(
(tx, current) => takeSnapshot(tx, current.conversationId, rt.registries.sections, rt.registries.tools),
ctx,
);
if (snapshot.settings.model === undefined) return failStep(task, "no_model", "no model configured");
const retry = (await rt.sticky(task.conversationId, ctx)).retry;
const warnings: string[] = [];
const { desired, tools } = await prepareDraft(
rt,
rt.registries.sections,
canonical,
seed,
snapshot.settings,
(message) => warnings.push(message),
ctx,
);
const model = snapshot.settings.model as unknown as ModelRef;
const thinkingLevel = snapshot.settings.thinkingLevel as ThinkingLevel;
return {
next: async (tx, current) => {
const c = current.conversationId;
const again = await takeSnapshot(tx, c, rt.registries.sections, rt.registries.tools);
if (!sameSnapshot(again.snapshot, snapshot)) return "retry";
const plan = await planManagedEntry(tx, c, snapshot, canonical, desired, tools, rt.now());
for (const message of warnings) tx.emit({ type: "warning", source: "generation", message });
const system = plan === undefined ? null : tx.appendEntry(c, { kind: "pi.system", ...plan });
const cutoff = system ?? (await tx.newestEntry(c))!.id;
tx.sticky(c).turn = { tools: [] }; // a new generation starts a fresh turn view
return {
phase: "prepared",
cutoff,
system,
model,
thinkingLevel,
tools: tools.map((t) => t.name),
retry,
attempt: 0,
};
},
};
},
phases: {
// Step 2: derive the request from the cutoff (hooks rerun), overflow check, write the
// in-flight checkpoint, call the provider. Reached fresh from `initial`/`retrying`,
// and again on recovery of `prepared` (step 1 never reruns).
async prepared(task, rt, ctx) {
const cp = task.checkpoint;
const model = rt.models.resolve(cp.model);
if (model === undefined)
return failStep(task, "no_model", `model ${cp.model.provider}/${cp.model.modelId} unavailable`);
const derived = await derive(task, cp, rt, ctx);
if (estimate(derived.messages) > model.contextWindow - model.maxTokens) return overflow(task, rt);
const attempt = cp.attempt + 1;
await rt.commit((tx) => {
tx.checkpoint({ ...cp, phase: "requesting", attempt });
tx.emit({ type: "generation.started", taskId: task.id, attempt });
}, ctx); // before the effect
const { terminal, deferred } = await stream(cp, model, derived.messages, rt, ctx);
if (deferred !== undefined) {
const pollAt = rt.now() + (deferred.pollAfterMs ?? 5000);
return {
next: (tx) => {
tx.emit({ type: "generation.deferred", taskId: task.id, pollAt });
return { ...cp, phase: "deferred", attempt, handle: toStored(deferred), pollAt };
},
};
}
return classify(task, { ...cp, attempt }, terminal!, rt, ctx);
},
// Only reached by the scheduler after a crash: the call may have happened and there
// is no result lookup. Count it as a failed attempt and retry per policy.
async requesting(task, rt) {
const cp = task.checkpoint;
const decision = retryDecision(cp, null, rt.now());
if (decision.kind === "fail") return failStep(task, decision.reason, "interrupted");
return {
next: (tx, current) => {
tx.appendEntry(current.conversationId, {
kind: "pi.usage",
data: { attempt: cp.attempt, error: "interrupted" },
});
tx.emit({
type: "generation.retrying",
taskId: current.id,
attempt: cp.attempt,
retryAt: decision.untilMs,
error: "interrupted",
});
return { ...cp, phase: "retrying", untilMs: decision.untilMs, lastError: "interrupted" };
},
};
},
// Durable backoff, then back to step 2. Resets the streaming view.
async retrying(task, rt, ctx) {
const cp = task.checkpoint;
await rt.sleep(cp.untilMs, ctx);
const { untilMs: _u, lastError: _e, ...prep } = cp;
return {
next: (tx, current) => {
tx.sticky(current.conversationId).turn.message = undefined;
return { ...prep, phase: "prepared" };
},
};
},
// Provider-side async. Poll; not the retry backoff. Same after a crash: the handle is durable.
async deferred(task, rt, ctx) {
const cp = task.checkpoint;
const model = rt.models.resolve(cp.model);
if (model === undefined) return failStep(task, "no_model", "model disappeared");
await rt.sleep(cp.pollAt, ctx);
const result = await rt.models.fetchDeferred(model, cp.handle, ctx);
if ("deferred" in result && result.deferred !== undefined) {
const pollAt = rt.now() + (result.deferred.pollAfterMs ?? 5000);
return {
next: (tx) => {
tx.emit({ type: "generation.deferred", taskId: task.id, pollAt });
return { ...cp, handle: toStored(result.deferred), pollAt };
},
};
}
const message = result as AssistantMessage;
await rt.commit((tx, current) => {
tx.sticky(current.conversationId).turn.message = toStored(message);
}, ctx);
const { handle: _handle, pollAt: _pollAt, ...prep } = cp;
return classify(task, prep, message, rt, ctx);
},
},
async abort(task, rt, ctx) {
const cp = task.checkpoint;
if (cp?.phase === "deferred") {
const model = rt.models.resolve(cp.model);
if (model) await rt.models.cancelDeferred(model, cp.handle as DeferredHandle, ctx).catch(() => {});
}
return async (tx, current) => {
const partial = tx.sticky(current.conversationId).turn.message;
let assistant: Id | undefined;
// Display-only: the partial goes in `data`, never in `model`, so it cannot enter a request.
if (partial && partial.content.length > 0)
assistant = tx.appendEntry(current.conversationId, {
kind: "pi.assistant",
data: {
attempt: cp?.attempt ?? 0,
display: { ...toStored(partial), stopReason: "aborted" },
reason: "aborted",
} satisfies DisplayAssistantData,
});
tx.sticky(current.conversationId).turn = { tools: [] };
await tx.resolveInputs(task.input.inputs, { status: "unanswered", reason: "aborted" });
tx.emit({ type: "turn.ended", inputs: task.input.inputs, status: "unanswered", reason: "aborted" });
return assistant === undefined ? {} : { assistant };
};
},
};
// ---------------------------------------------------------------------------
// Request derivation and streaming
// ---------------------------------------------------------------------------
async function derive(task: G, cp: Prep, rt: Rt, ctx: Context): Promise<{ messages: RequestMessage[] }> {
const { messages } = await rt.context(task.conversationId, cp.cutoff, ctx);
let request = { messages };
await rt.hooks.each(
ctx,
(h, api) => h.beforeRequest?.(request, { ...api, cutoff: cp.cutoff }, ctx),
(value) => {
if (value !== undefined) request = value;
},
);
return request;
}
/** Stream, coalescing frames into the turn view. Flush on size or time. */
async function stream(
cp: Prep,
model: NonNullable<ReturnType<Rt["models"]["resolve"]>>,
messages: RequestMessage[],
rt: Rt,
ctx: Context,
): Promise<{ terminal?: AssistantMessage; deferred?: DeferredHandle }> {
const encoder = new AssistantMessageFrameEncoder();
let pending: AssistantMessageFrame[] = [];
let pendingBytes = 0;
let lastFlush = rt.now();
let terminal: AssistantMessage | undefined;
let deferred: DeferredHandle | undefined;
let flushedContent = false;
const flush = async () => {
const batch = pending;
if (batch.length === 0) return;
pending = [];
pendingBytes = 0;
lastFlush = rt.now();
if (batch.some((frame) => "delta" in frame)) flushedContent = true;
await rt.commit((tx, current) => {
const turn = tx.sticky(current.conversationId).turn;
for (const f of batch) applyFrame(turn as { message?: AssistantMessage }, f);
}, ctx);
};
try {
const iterator = rt.models
.stream(model, { messages, thinkingLevel: cp.thinkingLevel }, ctx)
[Symbol.asyncIterator]();
let pull = iterator.next();
for (;;) {
const selected: { kind: "event"; result: IteratorResult<AssistantMessageEvent> } | { kind: "flush" } =
pending.length === 0
? { kind: "event", result: await pull }
: await Promise.race([
pull.then((result) => ({ kind: "event" as const, result })),
rt.sleep(lastFlush + 100, ctx).then(() => ({ kind: "flush" as const })),
]);
if (selected.kind === "flush") {
await flush();
continue;
}
if (selected.result.done) break;
const event = selected.result.value;
if (event.type === "done") {
encoder.encode(event);
if (event.reason === "deferred" && event.message.deferred !== undefined) deferred = event.message.deferred;
else terminal = event.message;
break;
}
if (event.type === "error") {
encoder.encode(event);
terminal = event.error;
break;
}
const frame = encoder.encode(event);
pull = iterator.next();
if (frame === undefined) continue;
pending.push(frame);
pendingBytes += "delta" in frame ? frame.delta.length : 64;
if (pendingBytes >= 256 || (!flushedContent && "delta" in frame)) await flush();
}
} catch (error) {
if (ctx.abortSignal?.aborted) throw error;
terminal = {
role: "assistant",
content: [],
api: model.api,
provider: model.provider,
model: model.id,
usage: emptyUsage(),
stopReason: "error",
errorMessage: String(error),
timestamp: rt.now(),
};
}
if (pending.length > 0) await flush();
if (terminal === undefined && deferred === undefined) throw new Error("stream ended without a terminal message");
return { terminal, deferred };
}
// ---------------------------------------------------------------------------
// Steps 3–5: classify the terminal message.
// ---------------------------------------------------------------------------
async function classify(task: G, cp: Prep, message: AssistantMessage, rt: Rt, ctx: Context): Promise<S> {
await rt.hooks.each(ctx, (h, api) => h.afterResponse?.(message, { ...api, attempt: cp.attempt }, ctx));
if (message.stopReason === "error") {
const decision = retryDecision(cp, message, rt.now());
if (decision.kind === "retry") {
return {
next: (tx, current) => {
tx.appendEntry(current.conversationId, {
kind: "pi.usage",
data: {
attempt: cp.attempt,
usage: toStored(message.usage),
error: message.errorMessage ?? "provider error",
},
});
tx.emit({
type: "generation.retrying",
taskId: current.id,
attempt: cp.attempt,
retryAt: decision.untilMs,
error: message.errorMessage ?? "provider error",
});
return {
...cp,
phase: "retrying",
untilMs: decision.untilMs,
lastError: message.errorMessage ?? "provider error",
};
},
};
}
return { done: terminalError(task, cp, message, decision.reason) };
}
if (message.stopReason === "aborted") return { done: terminalError(task, cp, message, "provider") };
const calls = message.content.filter((c): c is ToolCall => c.type === "toolCall");
const stored = toStored(message);
if (calls.length > 0) {
return {
done: async (tx, current) => {
const c = current.conversationId;
const collapseThrough = await thresholdCollapseThrough(tx, c, message); // read BEFORE the append
const assistant = tx.appendEntry(c, {
kind: "pi.assistant",
model: [stored],
data: { attempt: cp.attempt },
});
const turn = tx.sticky(c).turn;
turn.message = undefined;
turn.tools = calls.map((call) => ({
callId: call.id,
name: call.name,
args: toStored(call.arguments),
status: "pending",
}));
const tools = calls.map((call, index) =>
tx.createTask({ kind: "pi.tool", input: { assistant, call: toStored(call), offered: cp.tools, index } }),
);
const postTools = tx.createTask({
kind: "pi.post_tools",
after: tools,
input: { inputs: task.input.inputs, assistant, tools },
});
if (collapseThrough !== undefined)
tx.createTask({ kind: "pi.collapse", input: { reason: "threshold", through: collapseThrough } });
tx.emit({ type: "generation.completed", taskId: current.id, entry: assistant, toolCalls: calls.length });
return { status: "completed", result: { assistant, tools, postTools } };
},
};
}
let continueText: string | undefined;
await rt.hooks.each(
ctx,
(h, api) => h.onYield?.(message, api, ctx),
(value) => {
if (value === undefined) return;
continueText = value.continue;
return true;
},
);
return {
done: async (tx, current) => {
const c = current.conversationId;
const collapseThrough = await thresholdCollapseThrough(tx, c, message); // read BEFORE the append
const head = await tx.newestEntry(c, { withHead: true });
const assistant = tx.appendEntry(c, { kind: "pi.assistant", model: [stored], data: { attempt: cp.attempt } });
tx.emit({ type: "generation.completed", taskId: current.id, entry: assistant, toolCalls: 0 });
tx.sticky(c).turn = { tools: [] };
if (collapseThrough !== undefined)
tx.createTask({ kind: "pi.collapse", input: { reason: "threshold", through: collapseThrough } });
const { triggers, terminated } = await tx.boundary(c, "final", head?.id);
if (continueText !== undefined && triggers.length === 0 && !terminated) {
tx.appendEntry(c, {
kind: "pi.user",
model: [{ role: "user", content: continueText, timestamp: rt.now() }],
data: { continuation: true, from: assistant },
});
const successor = tx.createTask({ kind: "pi.generation", input: { inputs: task.input.inputs } });
return { status: "completed", result: { assistant, tools: [], successor } };
}
await tx.resolveInputs(task.input.inputs, { status: "done", answer: assistant });
tx.emit({ type: "turn.ended", inputs: task.input.inputs, status: "done", answer: assistant });
if (triggers.length > 0) {
const successor = tx.createTask({ kind: "pi.generation", input: { inputs: triggers } });
tx.emit({ type: "turn.started", inputs: triggers });
return { status: "completed", result: { assistant, tools: [], successor } };
}
return { status: "completed", result: { assistant, tools: [] } };
},
};
}
/** Provider error / aborted stop: a display-only assistant entry (no `model`), inputs unanswered. */
function terminalError(task: G, cp: Prep, message: AssistantMessage, reason: "provider" | "retries_exhausted"): Out {
return async (tx, current) => {
const head = await tx.newestEntry(current.conversationId, { withHead: true });
const assistant = tx.appendEntry(current.conversationId, {
kind: "pi.assistant",
data: {
attempt: cp.attempt,
display: toStored(message),
reason: message.stopReason === "aborted" ? "aborted" : "error",
} satisfies DisplayAssistantData,
});
tx.emit({
type: "generation.failed",
taskId: current.id,
reason,
detail: message.errorMessage ?? "provider error",
entry: assistant,
});
await settleFailedTurn(tx, current, task.input.inputs, message.errorMessage ?? "provider error", head?.id);
return fail(reason, message.errorMessage ?? "provider error", assistant);
};
}
async function settleFailedTurn(
tx: CoreTx,
current: Task,
inputs: readonly Id[],
detail: string,
headBoundary: Id | undefined,
): Promise<void> {
tx.sticky(current.conversationId).turn = { tools: [] };
await tx.resolveInputs(inputs, { status: "unanswered", reason: "failed", detail });
tx.emit({ type: "turn.ended", inputs: [...inputs], status: "unanswered", reason: "failed", detail });
const { triggers } = await tx.boundary(current.conversationId, "final", headBoundary);
if (triggers.length > 0) {
tx.createTask({ kind: "pi.generation", conversationId: current.conversationId, input: { inputs: triggers } });
tx.emit({ type: "turn.started", inputs: triggers });
}
}
type RetryDecision = { kind: "retry"; untilMs: number } | { kind: "fail"; reason: "provider" | "retries_exhausted" };
export function retryDecision(
cp: { retry: RetryPolicy; attempt: number },
message: AssistantMessage | null,
now: number,
): RetryDecision {
if (message !== null && !isRetryableAssistantError(message)) return { kind: "fail", reason: "provider" };
if (!cp.retry.enabled) return { kind: "fail", reason: "provider" };
if (cp.attempt > cp.retry.maxRetries) return { kind: "fail", reason: "retries_exhausted" };
const delay = cp.retry.baseDelayMs * 2 ** Math.max(0, cp.attempt - 1);
const safeDelay = Number.isSafeInteger(delay) ? delay : Number.MAX_SAFE_INTEGER;
return { kind: "retry", untilMs: now + Math.min(safeDelay, cp.retry.maxAgentDelayMs ?? 60_000) };
}
// ---------------------------------------------------------------------------
// Collapse triggers
// ---------------------------------------------------------------------------
/** Read-side of threshold collapse: computed before the assistant is appended, applied after. */
async function thresholdCollapseThrough(
tx: CoreTx,
conversationId: Id,
message: AssistantMessage,
): Promise<Id | undefined> {
if ((await tx.tasks({ conversationId, kind: "pi.collapse", status: ["pending", "running"] })).length > 0)
return undefined;
const state = tx.rewindable(conversationId);
if (state.threshold <= 0) return undefined;
const { entries, messages } = await tx.context(conversationId);
const used = Math.max(
(message.usage?.input ?? 0) + (message.usage?.output ?? 0),
estimate([...messages, toStored(message)]),
);
if (used <= state.threshold) return undefined;
return chooseThrough(entries, state.keepRecent);
}
function overflow(task: G, rt: Rt): S {
return {
done: async (tx, current) => {
const c = current.conversationId;
const state = tx.rewindable(c);
const { entries, head } = await tx.context(c);
const through = chooseThrough(entries, state.keepRecent);
tx.sticky(c).turn = { tools: [] };
if (through === undefined) {
await tx.write(c, {
kind: "pi.notice",
model: [{ role: "user", content: "Context too large; nothing to compact.", timestamp: rt.now() }],
});
tx.emit({
type: "generation.failed",
taskId: current.id,
reason: "overflow",
detail: "request exceeds context window and nothing is collapsible",
});
await settleFailedTurn(tx, current, task.input.inputs, "overflow", head?.id);
return fail("overflow", "request exceeds context window and nothing is collapsible");
}
const collapse = tx.createTask({ kind: "pi.collapse", input: { reason: "overflow", through } });
tx.createTask({ kind: "pi.generation", after: [collapse], input: { inputs: task.input.inputs } });
tx.emit({
type: "generation.failed",
taskId: current.id,
reason: "overflow",
detail: `collapsing through ${through}`,
});
return fail("overflow", `collapsing through ${through}`);
},
};
}

View file

@ -0,0 +1,187 @@
import type { Context } from "@earendil-works/chord";
import type { Completion, Kind, KindConfig, ProcessSpec, ProcessStatus, Runtime, Step, Task } from "../types.ts";
export type JobInput = { [K in keyof ProcessSpec]: ProcessSpec[K] } & {
notify: boolean;
rerun: boolean;
every?: number;
notBefore?: number;
};
export type JobCheckpoint =
| { phase: "waiting"; untilMs: number; occurrence: number }
| { phase: "spawning"; key: string; occurrence: number }
| { phase: "running"; key: string; occurrence: number };
export type JobOutput = {
stdout?: string;
stderr?: string;
droppedStdout?: number;
droppedStderr?: number;
exitCode?: number;
occurrence?: number;
};
export type JobResult = { exitCode: number; occurrences: number; stdout: string; stderr: string };
export type JobFailure = { reason: "spawn" | "interrupted"; detail: string };
type JobTask = Task<JobInput, JobCheckpoint>;
type JobStep = Step<JobCheckpoint, JobResult, JobFailure>;
const failed = (reason: JobFailure["reason"], detail: string): Completion<JobResult, JobFailure> => ({
status: "failed",
failure: { reason, detail },
});
const notice = (text: string, now: number) => ({
kind: "pi.notice",
model: [{ role: "user" as const, content: text, timestamp: now }],
});
const jobKind: Kind<
JobInput,
JobCheckpoint,
JobResult,
JobFailure,
{ killed: boolean },
object,
KindConfig,
JobOutput
> = {
name: "pi.job",
slot: () => ({}),
describe: (task) => ({
stage: task.checkpoint?.phase ?? "starting",
...(task.slot ?? {}),
}),
inflight: ["spawning", "running"],
phases: {
async waiting(task, runtime, ctx) {
await runtime.sleep(task.checkpoint.untilMs, ctx);
return spawn(task, task.checkpoint.occurrence, runtime, ctx);
},
async spawning(task, runtime, ctx) {
return reconcile(task, task.checkpoint, runtime, ctx);
},
async running(task, runtime, ctx) {
return reconcile(task, task.checkpoint, runtime, ctx);
},
},
async initial(task, runtime, ctx) {
if (runtime.processHost === undefined) return { done: () => failed("spawn", "no process host") };
const now = runtime.now();
const untilMs = task.input.notBefore ?? now;
if (untilMs > now) return { next: { phase: "waiting", untilMs, occurrence: 1 } };
return spawn(task, 1, runtime, ctx);
},
async abort(task, runtime, ctx) {
const checkpoint = task.checkpoint;
if (checkpoint === undefined || checkpoint.phase === "waiting" || runtime.processHost === undefined)
return async () => ({ killed: false });
await runtime.processHost.kill(checkpoint.key, "SIGTERM", ctx);
await runtime.sleep(runtime.now() + 5000, ctx);
await runtime.processHost.kill(checkpoint.key, "SIGKILL", ctx);
return async () => ({ killed: true });
},
};
export const job = Object.freeze(jobKind);
async function spawn(task: JobTask, occurrence: number, runtime: Runtime, ctx: Context): Promise<JobStep> {
const key = `${task.id}:${occurrence}`;
await runtime.commit((tx) => tx.checkpoint({ phase: "spawning", key, occurrence }), ctx);
try {
await runtime.processHost!.start(key, task.input, ctx);
} catch (error) {
if (ctx.abortSignal?.aborted) throw error;
return {
done: async (tx, current) => {
if (task.input.notify)
await tx.write(
current.conversationId,
notice(`job ${task.id} failed to start: ${String(error)}`, runtime.now()),
);
return failed("spawn", String(error));
},
};
}
await runtime.commit((tx) => tx.checkpoint({ phase: "running", key, occurrence }), ctx);
return poll(task, key, occurrence, runtime, ctx);
}
async function reconcile(
task: JobTask,
checkpoint: Extract<JobCheckpoint, { phase: "spawning" | "running" }>,
runtime: Runtime,
ctx: Context,
): Promise<JobStep> {
if (runtime.processHost === undefined) return { done: () => failed("interrupted", "no process host after restart") };
let status: ProcessStatus;
try {
status = await runtime.processHost.status(checkpoint.key, ctx);
} catch (error) {
return { done: () => failed("interrupted", `host status failed: ${String(error)}`) };
}
if (status.status === "unknown") {
if (!task.input.rerun) return { done: () => failed("interrupted", "process outcome unknown") };
return spawn(task, checkpoint.occurrence, runtime, ctx);
}
if (checkpoint.phase === "spawning") {
await runtime.commit(
(tx) => tx.checkpoint({ phase: "running", key: checkpoint.key, occurrence: checkpoint.occurrence }),
ctx,
);
}
return poll(task, checkpoint.key, checkpoint.occurrence, runtime, ctx);
}
async function poll(task: JobTask, key: string, occurrence: number, runtime: Runtime, ctx: Context): Promise<JobStep> {
for (let attempt = 0; ; attempt++) {
let status: ProcessStatus;
try {
status = await runtime.processHost!.status(key, ctx);
} catch (error) {
return { done: () => failed("interrupted", `host status failed: ${String(error)}`) };
}
if (status.status === "unknown") {
if (!task.input.rerun) return { done: () => failed("interrupted", "process outcome unknown") };
return spawn(task, occurrence, runtime, ctx);
}
const snapshot = status;
await runtime.commit((tx, current) => {
const output = tx.slot({ id: current.id, kind: jobKind });
output.stdout = snapshot.stdout;
output.stderr = snapshot.stderr;
output.droppedStdout = snapshot.droppedStdout;
output.droppedStderr = snapshot.droppedStderr;
if (snapshot.status === "exited") output.exitCode = snapshot.exitCode;
}, ctx);
if (snapshot.status === "exited") {
const { exitCode, stdout, stderr } = snapshot;
if (task.input.every === undefined) {
return {
done: async (tx, current) => {
if (task.input.notify)
await tx.write(
current.conversationId,
notice(`job ${task.id} exited with code ${exitCode}`, runtime.now()),
);
return { status: "completed", result: { exitCode, occurrences: occurrence, stdout, stderr } };
},
};
}
const untilMs = runtime.now() + task.input.every;
return {
next: async (tx, current) => {
if (task.input.notify)
await tx.write(
current.conversationId,
notice(`job ${task.id} occurrence ${occurrence} exited with code ${exitCode}`, runtime.now()),
);
const output = tx.slot({ id: current.id, kind: jobKind });
output.stdout = "";
output.stderr = "";
output.droppedStdout = 0;
output.droppedStderr = 0;
delete output.exitCode;
output.occurrence = occurrence + 1;
return { phase: "waiting", untilMs, occurrence: occurrence + 1 };
},
};
}
await runtime.sleep(runtime.now() + Math.min(1000, 100 * 2 ** Math.min(attempt, 4)), ctx);
}
}

View file

@ -0,0 +1,43 @@
import type { Context } from "@earendil-works/chord";
import { type JsonValue, type Kind, type Runtime, type Step, type Task, toStored } from "../types.ts";
import { taskApi } from "./task-api.ts";
export type PluginInput = { handler: string; input: JsonValue };
type PluginCheckpoint = { phase: "started" };
type PluginFailure = { reason: "missing_handler" | "threw"; detail: string };
const pluginKind: Kind<PluginInput, PluginCheckpoint, JsonValue, PluginFailure, null> = {
name: "pi.plugin",
inflight: ["started"],
phases: {
async started(task, runtime, ctx) {
return run(task, runtime, ctx);
},
},
async initial(task, runtime, ctx) {
await runtime.commit((tx) => tx.checkpoint({ phase: "started" }), ctx);
return run(task, runtime, ctx);
},
async abort() {
return async () => null;
},
};
export const plugin = Object.freeze(pluginKind);
async function run(
task: Task<PluginInput, PluginCheckpoint>,
runtime: Runtime,
ctx: Context,
): Promise<Step<PluginCheckpoint, JsonValue, PluginFailure>> {
const handler = runtime.plugins.get(task.input.handler);
if (handler === undefined) {
return { done: () => ({ status: "failed", failure: { reason: "missing_handler", detail: task.input.handler } }) };
}
try {
const result = toStored(await handler(task.input.input, taskApi(task, runtime), ctx));
return { done: () => ({ status: "completed", result }) };
} catch (error) {
if (ctx.abortSignal?.aborted) throw error;
return { done: () => ({ status: "failed", failure: { reason: "threw", detail: String(error) } }) };
}
}

View file

@ -0,0 +1,165 @@
import type { Context } from "@earendil-works/chord";
import type { AssistantMessage, ToolCall } from "@earendil-works/pi-ai";
import { type CoreKind, type HookInfo, type Id, type Stored, toStored } from "../types.ts";
import type { ToolTaskResult } from "./tool.ts";
export type PostToolsInput = { inputs: Id[]; assistant: Id; tools: Id[] };
export type PostToolsResult = { successor?: Id; ended?: "terminate" | "handoff" };
export interface PostToolsHooks {
afterTools(assistant: Id, results: Id[], info: HookInfo, ctx: Context): void | Promise<void>;
}
export const postToolsConfig = {
sticky: {
steeringMode: "one-at-a-time" as "all" | "one-at-a-time",
followUpMode: "one-at-a-time" as "all" | "one-at-a-time",
},
} as const;
export const postTools: CoreKind<
PostToolsInput,
never,
PostToolsResult,
never,
null,
PostToolsHooks,
typeof postToolsConfig
> = {
name: "pi.post_tools",
turn: true,
config: postToolsConfig,
phases: {},
async initial(task, rt, ctx) {
const rows = await rt.commit(async (tx) => {
const assistant = await tx.entry(task.input.assistant);
const calls = ((assistant?.model?.[0] as Stored<AssistantMessage> | undefined)?.content ?? []).filter(
(content): content is Stored<ToolCall> => content.type === "toolCall",
);
return Promise.all(
task.input.tools.map(async (id, index) => {
const toolTask = (await tx.task(id))!;
const call = calls[index]!;
if (toolTask.outcome?.status === "completed") {
const result = toolTask.outcome.result as ToolTaskResult;
return {
call,
entry: result.entry as Id | undefined,
control: result.control,
missing: undefined as "orphaned" | "aborted" | undefined,
};
}
if (toolTask.outcome?.status === "aborted") {
const aborted = toolTask.outcome.result as { entry?: Id };
return {
call,
entry: aborted.entry,
control: undefined,
missing: aborted.entry === undefined ? ("aborted" as const) : undefined,
};
}
return { call, entry: undefined, control: undefined, missing: "orphaned" as const };
}),
);
}, ctx);
const results = rows.map((row) => row.entry).filter((entry): entry is Id => entry !== undefined);
await rt.hooks.each(ctx, (hook, api) => hook.afterTools?.(task.input.assistant, results, api, ctx));
return {
done: async (tx, current) => {
const conversationId = current.conversationId;
const head = await tx.newestEntry(conversationId, { withHead: true });
const state = tx.rewindable(conversationId);
const selectedTools = [...state.selectedTools];
let terminate = false;
let handoff: string | undefined;
for (const row of rows) {
for (const name of row.control?.addTools ?? [])
if (!selectedTools.includes(name)) selectedTools.push(name);
if (row.control?.terminate) terminate = true;
if (row.control?.handoff !== undefined) handoff = row.control.handoff;
}
if (selectedTools.length !== state.selectedTools.length) {
state.selectedTools = selectedTools;
tx.emit({ type: "config.changed", keys: ["selectedTools"] });
}
for (const row of rows) {
if (row.missing === undefined) continue;
tx.appendEntry(conversationId, {
kind: "pi.tool_result",
model: [
toStored({
role: "toolResult" as const,
toolCallId: row.call.id,
toolName: row.call.name,
content: [{ type: "text" as const, text: `Tool result unavailable: task ${row.missing}.` }],
isError: true,
timestamp: rt.now(),
}),
],
data: { diagnostics: [{ severity: "error", message: row.missing, code: row.missing }] },
});
tx.emit({
type: "warning",
source: "post_tools",
message: `synthesized ${row.missing} tool result for ${row.call.id}`,
});
}
let headBoundary = head?.id;
if (handoff !== undefined) {
headBoundary = tx.appendEntry(conversationId, {
kind: "pi.handoff",
head: "self",
model: [{ role: "user", content: handoff, timestamp: rt.now() }],
});
}
if (handoff !== undefined || terminate) {
tx.sticky(conversationId).turn = { tools: [] };
await tx.resolveInputs(task.input.inputs, { status: "done", answer: task.input.assistant });
tx.emit({ type: "turn.ended", inputs: task.input.inputs, status: "done", answer: task.input.assistant });
const { triggers } = await tx.boundary(conversationId, "final", headBoundary);
if (triggers.length > 0) tx.emit({ type: "turn.started", inputs: triggers });
const ended = handoff !== undefined ? "handoff" : "terminate";
return {
status: "completed",
result:
triggers.length > 0
? { ended, successor: tx.createTask({ kind: "pi.generation", input: { inputs: triggers } }) }
: { ended },
};
}
const { triggers, terminated } = await tx.boundary(conversationId, "postTools", headBoundary);
if (terminated) {
tx.sticky(conversationId).turn = { tools: [] };
await tx.resolveInputs(task.input.inputs, { status: "unanswered", reason: "terminated" });
tx.emit({ type: "turn.ended", inputs: task.input.inputs, status: "unanswered", reason: "terminated" });
if (triggers.length > 0) tx.emit({ type: "turn.started", inputs: triggers });
return {
status: "completed",
result:
triggers.length > 0
? { successor: tx.createTask({ kind: "pi.generation", input: { inputs: triggers } }) }
: {},
};
}
return {
status: "completed",
result: {
successor: tx.createTask({
kind: "pi.generation",
input: { inputs: [...task.input.inputs, ...triggers] },
}),
},
};
},
};
},
async abort(task) {
return async (tx, current) => {
tx.sticky(current.conversationId).turn = { tools: [] };
await tx.resolveInputs(task.input.inputs, { status: "unanswered", reason: "aborted" });
tx.emit({ type: "turn.ended", inputs: task.input.inputs, status: "unanswered", reason: "aborted" });
return null;
};
},
};

View file

@ -0,0 +1,52 @@
import type { Context } from "@earendil-works/chord";
import type { AnyKind, Id, JsonValue, Runtime, Task, TaskRef, ToolApi } from "../types.ts";
export function taskApi(task: Task, runtime: Runtime): ToolApi {
const isKind = (value: unknown): value is AnyKind =>
typeof value === "object" && value !== null && "name" in value && "initial" in value;
return {
taskId: task.id,
conversationId: task.conversationId,
callId: "",
stream: () => {
throw new Error("stream is only available to tools");
},
progress: () => {
throw new Error("progress is only available to tools");
},
memo: (() => Promise.reject(new Error("memo is only available to tools"))) as ToolApi["memo"],
async conversation(spec, ctx) {
const id = await runtime.createOwnedConversation(spec, ctx);
return {
id,
send: async (input, childContext) => {
const inputId = await runtime.sendOwned(id, input, childContext);
return {
id: inputId,
wait: (waitContext) => runtime.waitForInput(inputId, waitContext),
result: (resultContext) => runtime.commit((tx) => tx.input(inputId), resultContext),
};
},
abort: (childContext) => runtime.abortConversation(id, childContext),
};
},
task: ((
kind: AnyKind,
input: JsonValue,
opts: { background?: true; after?: Id[] },
ctx: Context,
): Promise<TaskRef> => {
if (!isKind(kind)) throw new Error("task(): pass a kind token");
return runtime.commit((tx) => tx.createTask(kind, input as never, opts), ctx);
}) as ToolApi["task"],
getTask: async (ref, ctx) => (await runtime.commit((tx) => tx.task(ref.id), ctx)) as never,
waitForTask: (ref, ctx) => runtime.waitForTask(ref.id, ctx) as never,
slot: (ref, ctx) =>
runtime.commit(async (tx) => {
const storedTask = await tx.task(ref.id);
if (storedTask === undefined) return undefined;
return (tx.snapshot({ doc: "sticky", conversationId: storedTask.conversationId }).tasks[ref.id] ??
undefined) as never;
}, ctx),
};
}

View file

@ -0,0 +1,479 @@
import type { Context } from "@earendil-works/chord";
import type { ToolCall } from "@earendil-works/pi-ai";
import type { TSchema } from "typebox";
import { Errors } from "typebox/value";
import { Bounded } from "../bounded.ts";
import {
type AnyToolDeclaration,
type BeforeToolApi,
type Closure,
type Completion,
type CoreKind,
type CoreTx,
type HookInfo,
type HookResult,
type Id,
type JsonObject,
type JsonValue,
type Runtime,
type Step,
type Stored,
type Task,
type ToolApi,
type ToolControl,
type ToolResult,
type ToolSlot,
toStored,
} from "../types.ts";
import { taskApi } from "./task-api.ts";
export type StoredToolCall = Stored<ToolCall>;
export type ToolInput = {
assistant: Id;
call: StoredToolCall;
offered: string[];
index: number; // position of the call in the assistant message; also this tool's slot in sticky.turn.tools
};
export type ToolCheckpoint = { phase: "started"; replay: "safe" | "unsafe"; call: StoredToolCall };
export type ToolTaskResult = { entry: Id; control?: ToolControl };
/** Hook points this kind calls. Register with `h.hooks(namespace, kinds.tool, { … })`. */
export interface ToolHooks {
/** Before invocation, before `started`. Chain on `call`; `{ block }` stops it. A throw blocks. */
beforeTool(
call: StoredToolCall,
api: BeforeToolApi,
ctx: Context,
): HookResult<{ call?: StoredToolCall; block?: string }>;
/** After the tool returns; may replace the result. Chain. */
afterTool(
call: StoredToolCall,
result: ToolResult,
api: HookInfo & { readonly callId: string },
ctx: Context,
): HookResult<ToolResult>;
}
type T = Task<ToolInput, ToolCheckpoint>;
type Rt = Runtime<ToolHooks, CoreTx>;
type Out = Closure<ToolTaskResult, never, CoreTx>;
const synthetic = (text: string, code: string): ToolResult => ({
content: [{ type: "text", text }],
isError: true,
diagnostics: [{ severity: "error", message: text, code }],
});
/** Validate arguments with the tool's real schema (TypeBox 1.x). */
export function invalid(declaration: AnyToolDeclaration, call: StoredToolCall): string | undefined {
const errors = [...Errors(declaration.parameters as TSchema, call.arguments)];
return errors.length === 0 ? undefined : errors.map((e) => `${e.instancePath || "/"}: ${e.message}`).join("; ");
}
const sameIdentity = (a: StoredToolCall, b: StoredToolCall) =>
a.id === b.id && a.name === b.name && a.namespace === b.namespace;
export const tool: CoreKind<ToolInput, ToolCheckpoint, ToolTaskResult, never, { entry: Id }, ToolHooks> = {
name: "pi.tool",
turn: true,
inflight: ["started"],
// Offered-set check, lookup, validate, beforeTool, validate again; then write `started`
// (the in-flight checkpoint) and invoke. A crash before `started` reruns all of this.
async initial(task, rt, ctx) {
const call = task.input.call;
if (!task.input.offered.includes(call.name))
return {
done: close(task, synthetic(`tool ${call.name} was not offered`, "not_offered"), undefined, rt.now()),
};
const declaration = rt.tools.get(call.name);
if (declaration === undefined)
return {
done: close(task, synthetic(`tool ${call.name} is not registered`, "missing_tool"), undefined, rt.now()),
};
const bad = invalid(declaration, call);
if (bad !== undefined)
return {
done: close(task, synthetic(`invalid arguments: ${bad}`, "invalid_arguments"), declaration, rt.now()),
};
const hooked = await beforeTool(task, rt, call, ctx);
if ("block" in hooked)
return { done: close(task, synthetic(`blocked: ${hooked.block}`, "blocked"), declaration, rt.now()) };
if (!sameIdentity(call, hooked.call))
return { done: close(task, synthetic("blocked: call identity changed", "blocked"), declaration, rt.now()) };
const badFinal = invalid(declaration, hooked.call);
if (badFinal !== undefined)
return {
done: close(
task,
synthetic(`invalid arguments after hook: ${badFinal}`, "invalid_arguments"),
declaration,
rt.now(),
),
};
const final = toStored(hooked.call);
await rt.commit((tx) => {
tx.checkpoint({ phase: "started", replay: declaration.replay ?? "unsafe", call: final });
const slot = tx.toolSlot(task);
slot.status = "running";
delete slot.waitingOn;
tx.emit({ type: "tool.started", taskId: task.id, callId: final.id, name: final.name });
}, ctx); // durable before the effect
return { done: await invoke(task, final, declaration, rt, ctx) };
},
phases: {
// Only entered by the scheduler after reopen. The stored final call is the evidence; beforeTool never reruns.
async started(task, rt, ctx) {
const cp = task.checkpoint;
const declaration = rt.tools.get(cp.call.name);
if (declaration === undefined)
return {
done: close(
task,
synthetic(`tool ${cp.call.name} unavailable after restart`, "unavailable"),
undefined,
rt.now(),
cp.call,
),
};
if (cp.replay !== "safe" || (declaration.replay ?? "unsafe") !== "safe")
return {
done: close(
task,
synthetic(`tool ${cp.call.name} was interrupted`, "interrupted"),
declaration,
rt.now(),
cp.call,
),
};
if (invalid(declaration, cp.call) !== undefined)
return {
done: close(
task,
synthetic(`tool ${cp.call.name} was interrupted; arguments no longer validate`, "interrupted"),
declaration,
rt.now(),
cp.call,
),
};
return { done: await invoke(task, cp.call, declaration, rt, ctx) };
},
},
async abort(task, rt, ctx) {
for (const conversationId of task.owns) {
const children = await rt.commit((tx) => tx.tasks({ conversationId, status: ["pending", "running"] }), ctx);
for (const child of children) if (!child.background) await rt.abortTask(child.id, ctx);
}
return (tx, current) => {
const call = task.checkpoint?.call ?? task.input.call;
const entry = tx.appendEntry(current.conversationId, {
kind: "pi.tool_result",
model: [toMessage(call, synthetic("tool aborted", "aborted"), rt.now())],
data: { diagnostics: [{ severity: "error", message: "aborted", code: "aborted" }] },
});
const slot = tx.sticky(current.conversationId).turn.tools[task.input.index];
if (slot !== undefined) {
slot.status = "aborted";
slot.entry = entry;
delete slot.waitingOn;
}
tx.emit({ type: "tool.aborted", taskId: current.id, callId: call.id, entry });
return { entry };
};
},
};
/** The one point where a throwing handler blocks rather than being skipped. */
async function beforeTool(
task: T,
rt: Rt,
call: StoredToolCall,
ctx: Context,
): Promise<{ call: StoredToolCall } | { block: string }> {
let current = call;
for (const binding of rt.hooks.handlers()) {
function memo<T extends JsonValue>(name: string, candidate: T, context: Context): Promise<T>;
function memo<T extends JsonValue>(name: string, context: Context): Promise<T | undefined>;
function memo<T extends JsonValue>(
name: string,
candidateOrContext: T | Context,
context?: Context,
): Promise<T | undefined> {
const targetContext = context ?? (candidateOrContext as Context);
return rt.commit((tx) => {
tx.plugins(binding.namespace);
const slot = tx.toolSlot(task);
if (!Object.hasOwn(slot, "memos")) slot.memos = {};
const memos = slot.memos!;
const key = `hook:${binding.namespace.id}:${name}`;
if (Object.hasOwn(memos, key)) return structuredClone(memos[key]) as T;
if (context === undefined) return undefined;
const candidate = structuredClone(candidateOrContext as T);
memos[key] = candidate;
return candidate;
}, targetContext);
}
const api: BeforeToolApi = {
...binding.api,
callId: call.id,
waiting: (context) =>
rt.commit((tx, currentTask) => {
tx.plugins(binding.namespace);
const slot = tx.toolSlot(currentTask as T);
if (slot.waitingOn === binding.namespace.id) return;
slot.waitingOn = binding.namespace.id;
tx.emit({
type: "tool.waiting",
taskId: currentTask.id,
callId: call.id,
on: binding.namespace.id,
});
}, context),
memo,
emit: (name, data, context) => rt.commit((tx) => tx.emit(binding.namespace, name, data), context),
};
try {
const value = await binding.handlers.beforeTool?.(current, api, ctx);
if (!value) continue;
if (value.block !== undefined) return { block: value.block };
if (value.call !== undefined) current = value.call;
} catch (error) {
if (ctx.abortSignal?.aborted) throw error;
return { block: `hook threw: ${String(error)}` };
}
}
return { call: current };
}
export const DEFAULT_BOUNDS = { maxBytes: 64 * 1024, maxLines: 200, retain: "head" as const };
async function invoke(
task: T,
call: StoredToolCall,
declaration: AnyToolDeclaration,
rt: Rt,
ctx: Context,
): Promise<Out> {
// The kernel owns the stream: one bounded buffer, one throttle, one flush path. Flush failures are not swallowed.
const b = { ...DEFAULT_BOUNDS, ...declaration.output };
const buffer = new Bounded(b.maxBytes, b.maxLines, b.retain);
let streamed = false;
let lastFlush = 0;
let flushing: Promise<void> = Promise.resolve();
let flushError: unknown;
const flush = () => {
lastFlush = rt.now();
const text = buffer.text();
flushing = flushing
.then(() =>
rt.commit((tx) => {
tx.toolSlot(task).output = text;
}, ctx),
)
.catch((e) => {
flushError ??= e;
});
};
const api = {
...taskApi(task, rt),
// Only the free fields; identity (callId/name/args/output) is protected.
progress: (update: (slot: Pick<ToolSlot, "progress" | "details" | "continuedBy">) => void, c: Context) =>
rt.commit((tx) => {
const slot = tx.toolSlot(task);
const free: Pick<ToolSlot, "progress" | "details" | "continuedBy"> = {
progress: slot.progress,
details: slot.details,
continuedBy: slot.continuedBy,
};
update(free);
if (free.progress !== undefined) slot.progress = free.progress;
else delete slot.progress;
if (free.details !== undefined) slot.details = toStored(free.details);
else delete slot.details;
if (free.continuedBy !== undefined) slot.continuedBy = free.continuedBy;
else delete slot.continuedBy;
}, c),
stream(chunk: string | Uint8Array) {
streamed = true;
buffer.push(typeof chunk === "string" ? new TextEncoder().encode(chunk) : chunk);
if (rt.now() - lastFlush >= 100) flush();
},
};
function memo<T extends JsonValue>(name: string, candidate: T, context: Context): Promise<T>;
function memo<T extends JsonValue>(name: string, context: Context): Promise<T | undefined>;
function memo<T extends JsonValue>(
name: string,
candidateOrContext: T | Context,
context?: Context,
): Promise<T | undefined> {
const targetContext = context ?? (candidateOrContext as Context);
return rt.commit((tx) => {
const slot = tx.toolSlot(task);
if (!Object.hasOwn(slot, "memos")) slot.memos = {};
const memos = slot.memos!;
const key = `tool:${name}`;
if (Object.hasOwn(memos, key)) return structuredClone(memos[key]) as T;
if (context === undefined) return undefined;
const candidate = structuredClone(candidateOrContext as T);
memos[key] = candidate;
return candidate;
}, targetContext);
}
const toolApi: typeof api & Pick<ToolApi, "callId" | "memo"> = { ...api, callId: call.id, memo };
let result: ToolResult;
try {
result = await declaration.execute(call.arguments as never, toolApi, ctx);
} catch (error) {
if (ctx.abortSignal?.aborted) throw error;
result = synthetic(`tool threw: ${String(error)}`, "threw");
}
if (streamed) {
flush();
await flushing;
if (flushError !== undefined) throw flushError; // a persistence failure is never hidden
if (result.content === undefined)
result = {
...result,
content: [{ type: "text", text: buffer.text() }],
...(buffer.dropped
? {
diagnostics: [
...(result.diagnostics ?? []),
{
severity: "warn" as const,
code: "truncated",
message: `${buffer.droppedBytes} bytes / ${buffer.droppedLines} lines dropped (${b.retain} ${b.maxBytes} retained)`,
},
],
}
: {}),
};
}
let final = result;
await rt.hooks.each(
ctx,
(h, api) => h.afterTool?.(call, final, { ...api, callId: call.id }, ctx),
(value) => {
if (value !== undefined) final = value;
},
);
return close(
task,
final,
declaration,
rt.now(),
call,
streamed ? { bytes: buffer.droppedBytes, lines: buffer.droppedLines } : undefined,
);
}
/** Closure: bound the output, store the exact model message under the stored final call, rest as strict-JSON data. */
function close(
task: T,
raw: ToolResult,
declaration: AnyToolDeclaration | undefined,
now: number,
call: StoredToolCall = task.input.call,
streamTruncated?: { bytes: number; lines: number },
): Out {
return (tx, current) => {
const bounded = bound({ ...raw, content: raw.content ?? [] }, declaration?.output);
const result = bounded.result;
const bytes = (bounded.truncated?.bytes ?? 0) + (streamTruncated?.bytes ?? 0);
const lines = (bounded.truncated?.lines ?? 0) + (streamTruncated?.lines ?? 0);
const truncated = bytes > 0 || lines > 0 ? { bytes, lines } : undefined;
const data: JsonObject = {};
if (result.details !== undefined) data.details = toStored(result.details);
if (result.diagnostics !== undefined) data.diagnostics = toStored(result.diagnostics);
if (result.control !== undefined) data.control = toStored(result.control);
if (truncated !== undefined) data.truncated = truncated;
const entry = tx.appendEntry(current.conversationId, {
kind: "pi.tool_result",
model: [toMessage(call, result, now)],
data,
});
const slot = tx.toolSlot(task);
slot.status = result.isError === true ? "error" : "done";
slot.entry = entry;
delete slot.waitingOn;
tx.emit({
type: "tool.finished",
taskId: current.id,
callId: call.id,
entry,
isError: result.isError === true,
...(result.control === undefined ? {} : { control: result.control }),
});
if (truncated !== undefined)
tx.emit({
type: "warning",
source: "tool",
message: `tool output truncated: ${truncated.bytes} bytes / ${truncated.lines} lines dropped`,
});
const completion: Completion<ToolTaskResult, never> = {
status: "completed",
result: { entry, ...(result.control === undefined ? {} : { control: toStored(result.control) }) },
};
return completion;
};
}
function toMessage(call: StoredToolCall, result: ToolResult, now: number) {
return toStored({
role: "toolResult" as const,
toolCallId: call.id,
toolName: call.name,
content: result.content ?? [],
isError: result.isError ?? false,
timestamp: now,
});
}
function bound(
result: ToolResult,
declared: AnyToolDeclaration["output"],
): { result: ToolResult; truncated?: { bytes: number; lines: number } } {
const bounds = { ...DEFAULT_BOUNDS, ...declared };
const blocks = result.content ?? [];
let text = blocks
.filter((block) => block.type === "text")
.map((block) => block.text)
.join("");
let droppedLines = 0;
let droppedBytes = 0;
const lines = text.split("\n");
if (lines.length > bounds.maxLines) {
droppedLines = lines.length - bounds.maxLines;
text = (bounds.retain === "head" ? lines.slice(0, bounds.maxLines) : lines.slice(-bounds.maxLines)).join("\n");
}
const bytes = new TextEncoder().encode(text);
if (bytes.length > bounds.maxBytes) {
droppedBytes = bytes.length - bounds.maxBytes;
text = new TextDecoder().decode(
bounds.retain === "head" ? bytes.subarray(0, bounds.maxBytes) : bytes.subarray(-bounds.maxBytes),
);
}
if (droppedBytes === 0 && droppedLines === 0) return { result };
let textIndex = -1;
for (let index = 0; index < blocks.length; index++) {
if (blocks[index]!.type !== "text") continue;
if (bounds.retain === "head" && textIndex >= 0) continue;
textIndex = index;
}
const content: NonNullable<ToolResult["content"]> = [];
for (let index = 0; index < blocks.length; index++) {
const block = blocks[index]!;
if (block.type !== "text") content.push(block);
else if (index === textIndex) content.push({ ...block, text });
}
const note = {
severity: "warn" as const,
code: "truncated",
message: `output truncated: ${droppedLines} lines, ${droppedBytes} bytes dropped`,
};
return {
result: { ...result, content, diagnostics: [...(result.diagnostics ?? []), note] },
truncated: { bytes: droppedBytes, lines: droppedLines },
};
}
export type { Step };

View file

@ -0,0 +1,123 @@
/**
* A transaction-scoped revocable membrane over a Chord-tracked document.
*
* Every object reached through a document proxy is wrapped lazily; all wrappers of one
* transaction share one liveness flag; identity is preserved by a WeakMap. Mutations forward
* to the underlying Chord proxy so its tracker records ops. At transaction finish — success,
* callback failure, validation failure, storage failure — `revoke()` flips the flag and every
* retained wrapper, root or nested, throws on any operation.
*
* Assigning one wrapper into another (`a.list = b.list`) is rejected: Chord stores the value it
* is given, and a stored proxy is the classic footgun. Assign plain values; splice in place.
*/
export class Membrane {
private alive = true;
private readonly wrappers = new WeakMap<object, object>();
private readonly isWrapper = new WeakSet<object>();
private readonly what: string;
constructor(what: string) {
this.what = what;
}
revoke() {
this.alive = false;
}
wrap<T extends object>(target: T): T {
const hit = this.wrappers.get(target);
if (hit !== undefined) return hit as T;
const m = this;
const dead = () => new TypeError(`document proxy (${m.what}) used outside its transaction`);
const out = (v: unknown): unknown => (typeof v === "object" && v !== null ? m.wrap(v) : v);
const inp = (value: unknown): unknown => {
if (typeof value !== "object" || value === null) return value;
m.assertPlainInput(value, new WeakSet());
return JSON.parse(JSON.stringify(value)) as unknown;
};
const proxy = new Proxy(target, {
get(t, key, receiver) {
if (!m.alive) throw dead();
const v = Reflect.get(t, key, receiver === proxy ? t : receiver);
if (typeof v === "function") {
// Methods run against the tracked target. Callback arguments (array elements and
// the array itself) must cross back through the membrane before user code sees them.
return (...args: unknown[]) => {
if (!m.alive) throw dead();
const safeArgs = args.map((arg) => {
if (typeof arg !== "function") return inp(arg);
return function (this: unknown, ...callbackArgs: unknown[]) {
if (!m.alive) throw dead();
return Reflect.apply(arg, this, callbackArgs.map(out));
};
});
return out(Reflect.apply(v, t, safeArgs));
};
}
return out(v);
},
set(t, key, value) {
if (!m.alive) throw dead();
if (key === "__proto__") throw new TypeError(`document proxy (${m.what}) cannot change prototypes`);
return Reflect.set(t, key, inp(value) as never);
},
deleteProperty(t, key) {
if (!m.alive) throw dead();
return Reflect.deleteProperty(t, key);
},
has(t, key) {
if (!m.alive) throw dead();
return Reflect.has(t, key);
},
ownKeys(t) {
if (!m.alive) throw dead();
return Reflect.ownKeys(t);
},
getOwnPropertyDescriptor(t, key) {
if (!m.alive) throw dead();
const d = Reflect.getOwnPropertyDescriptor(t, key);
if (d === undefined) return undefined;
// Descriptors must not leak a raw nested object.
if ("value" in d) (d as { value: unknown }).value = out(d.value);
return d;
},
defineProperty(t, key, descriptor) {
if (!m.alive) throw dead();
if ("get" in descriptor || "set" in descriptor)
throw new TypeError(`document proxy (${m.what}) cannot define accessors`);
const safe = "value" in descriptor ? { ...descriptor, value: inp(descriptor.value) } : descriptor;
return Reflect.defineProperty(t, key, safe);
},
getPrototypeOf(t) {
if (!m.alive) throw dead();
return Reflect.getPrototypeOf(t);
},
setPrototypeOf() {
if (!m.alive) throw dead();
throw new TypeError(`document proxy (${m.what}) cannot change prototypes`);
},
isExtensible(t) {
if (!m.alive) throw dead();
return Reflect.isExtensible(t);
},
preventExtensions() {
if (!m.alive) throw dead();
throw new TypeError(`document proxy (${m.what}) cannot change extensibility`);
},
});
this.wrappers.set(target, proxy);
this.isWrapper.add(proxy);
return proxy as T;
}
private assertPlainInput(value: object, seen: WeakSet<object>): void {
if (this.isWrapper.has(value))
throw new TypeError(`assigning a document proxy into a document (${this.what}); assign a plain value`);
if (seen.has(value)) throw new TypeError(`assigning a cyclic value into a document (${this.what})`);
seen.add(value);
for (const nested of Object.values(value)) {
if (typeof nested === "object" && nested !== null) this.assertPlainInput(nested, seen);
}
seen.delete(value);
}
}

View file

@ -0,0 +1,278 @@
import type { Context } from "@earendil-works/chord";
import { applyImmutable, isBase, type Op } from "@earendil-works/chord/delta";
import type {
Conversation,
DocRef,
Entry,
EntryScan,
Id,
Input,
JsonObject,
Seq,
Storage,
Task,
TaskScan,
Write,
} from "./types.ts";
/**
* In-memory Storage. The reference backend, and the read path of JsonlStorage.
*
* A batch is validated and staged in full before any table changes: one failed
* batch changes no table, no document, no ID high-water, no sequence. Committed
* IDs are never reused; IDs minted but never committed may be reused after reopen.
*/
export class MemoryStorage implements Storage {
protected readonly conversationsById = new Map<Id, Conversation>();
protected readonly entriesById = new Map<Id, Entry>();
protected readonly entriesByConversation = new Map<Id, Entry[]>(); // ascending
protected readonly tasksById = new Map<Id, Task>();
protected readonly inputsById = new Map<Id, Input>();
protected readonly inputsByRequest = new Map<string, Input>();
/** Rewindable docs keep their full op history with the seq of each batch, for `docAsOf`. */
protected readonly rewindableLog = new Map<Id, { seq: Seq; ops: Op[] }[]>();
protected readonly stickyLog = new Map<Id, Op[][]>();
protected sessionDoc: JsonObject | undefined;
protected readonly entrySeq = new Map<Id, Seq>(); // entry id → seq of its batch
protected nextIdValue = 1;
protected seq: Seq = 0;
private closed = false;
mintId(): Id {
return this.nextIdValue++;
}
protected setNextId(n: Id) {
this.nextIdValue = Math.max(this.nextIdValue, n);
}
async commit(writes: readonly Write[], _ctx: Context): Promise<Seq> {
if (this.closed) throw new Error("storage closed");
const seq = this.seq + 1;
const staged = this.stage(writes, seq); // throws before any mutation
staged();
this.seq = seq;
return seq;
}
/** Validate the whole batch against the current tables; return a function that applies it. */
protected stage(writes: readonly Write[], seq: Seq): () => void {
const ops: (() => void)[] = [];
const created = new Set<Id>();
let maxId = 0;
const claim = (id: Id, what: string) => {
if (!Number.isSafeInteger(id) || id <= 0) throw new Error(`${what}: invalid id ${id}`);
if (created.has(id)) throw new Error(`${what}: id ${id} created twice in one batch`);
created.add(id);
maxId = Math.max(maxId, id);
};
for (const w of writes) {
switch (w.type) {
case "conversation": {
if (this.conversationsById.has(w.conversation.id))
throw new Error(`conversation ${w.conversation.id} exists`);
claim(w.conversation.id, "conversation");
const c = clone(w.conversation);
ops.push(() => {
this.conversationsById.set(c.id, c);
this.entriesByConversation.set(c.id, []);
});
break;
}
case "entry": {
if (this.entriesById.has(w.entry.id)) throw new Error(`entry ${w.entry.id} exists`);
claim(w.entry.id, "entry");
const e = clone(w.entry);
ops.push(() => {
this.entriesById.set(e.id, e);
(
this.entriesByConversation.get(e.conversationId) ??
this.entriesByConversation.set(e.conversationId, []).get(e.conversationId)!
).push(e);
this.entrySeq.set(e.id, seq);
});
break;
}
case "task": {
if (this.tasksById.has(w.task.id)) throw new Error(`task ${w.task.id} exists`);
claim(w.task.id, "task");
const t = clone(w.task);
ops.push(() => this.tasksById.set(t.id, t));
break;
}
case "task.patch": {
const prev =
this.tasksById.get(w.patch.id) ??
[...writes]
.filter((x): x is Extract<Write, { type: "task" }> => x.type === "task")
.find((x) => x.task.id === w.patch.id)?.task;
if (prev === undefined) throw new Error(`patch for unknown task ${w.patch.id}`);
const p = clone(w.patch);
ops.push(() => {
const cur = this.tasksById.get(p.id)!;
const { id: _i, ...fields } = p;
const next = { ...cur, ...fields } as Task;
if (p.checkpoint === null) delete (next as { checkpoint?: unknown }).checkpoint;
this.tasksById.set(p.id, next);
});
break;
}
case "input": {
const i = clone(w.input);
if (!this.inputsById.has(i.id)) claim(i.id, "input");
ops.push(() => {
this.inputsById.set(i.id, i);
if (i.requestId !== undefined) this.inputsByRequest.set(`${i.conversationId}:${i.requestId}`, i);
});
break;
}
case "doc": {
const ref = clone(w.ref);
const o = clone(w.ops);
if (ref.doc === "session")
ops.push(() => {
this.sessionDoc = applyImmutable(
(this.sessionDoc ?? { plugins: {} }) as JsonObject,
o,
) as JsonObject;
});
else if (ref.doc === "rewindable")
ops.push(() =>
(
this.rewindableLog.get(ref.conversationId) ??
this.rewindableLog.set(ref.conversationId, []).get(ref.conversationId)!
).push({ seq, ops: o }),
);
else
ops.push(() =>
(
this.stickyLog.get(ref.conversationId) ??
this.stickyLog.set(ref.conversationId, []).get(ref.conversationId)!
).push(o),
);
break;
}
}
}
return () => {
for (const op of ops) op();
this.setNextId(maxId + 1);
};
}
async conversation(id: Id, _ctx?: Context) {
return optionalClone(this.conversationsById.get(id));
}
async conversations(_ctx?: Context) {
return [...this.conversationsById.values()].map(clone);
}
async entries(ids: readonly Id[], _ctx?: Context) {
const out = new Map<Id, Entry>();
for (const id of ids) {
const e = this.entriesById.get(id);
if (e) out.set(id, clone(e));
}
return out;
}
/** Newest-first, fork-aware: after this conversation's own entries, the parent's up to the fork point, and so on. */
async scanEntries(scan: EntryScan, _ctx?: Context) {
const out: Entry[] = [];
let conversationId: Id | undefined = scan.conversationId;
let cap: Id | undefined = scan.before;
while (conversationId !== undefined && out.length < scan.limit) {
const own = this.entriesByConversation.get(conversationId) ?? [];
for (let i = own.length - 1; i >= 0 && out.length < scan.limit; i--) {
const e = own[i]!;
if (cap !== undefined && e.id >= cap) continue;
if (scan.kind !== undefined && e.kind !== scan.kind) continue;
if (scan.withHead && e.head === undefined) continue;
out.push(clone(e));
}
const c = this.conversationsById.get(conversationId);
conversationId = c?.parent?.conversationId;
cap = c?.parent === undefined ? undefined : Math.min(cap ?? Infinity, c.parent.at + 1);
}
return out;
}
async task(id: Id, _ctx?: Context) {
return optionalClone(this.tasksById.get(id));
}
async scanTasks(scan: TaskScan, _ctx?: Context) {
const out: Task[] = [];
for (const t of this.tasksById.values()) {
if (scan.conversationId !== undefined && t.conversationId !== scan.conversationId) continue;
if (scan.status !== undefined && !scan.status.includes(t.status)) continue;
if (scan.kind !== undefined && t.kind !== scan.kind) continue;
out.push(clone(t));
}
return out;
}
async input(id: Id, _ctx?: Context) {
return optionalClone(this.inputsById.get(id));
}
async inputByRequest(conversationId: Id, requestId: string, _ctx?: Context) {
return optionalClone(this.inputsByRequest.get(`${conversationId}:${requestId}`));
}
async doc(ref: DocRef, _ctx?: Context): Promise<JsonObject | undefined> {
if (ref.doc === "session")
return this.sessionDoc === undefined ? { plugins: {} } : structuredClone(this.sessionDoc);
const log =
ref.doc === "rewindable"
? this.rewindableLog.get(ref.conversationId)?.map((r) => r.ops)
: this.stickyLog.get(ref.conversationId);
if (log === undefined) return undefined;
return clone(this.fold(log));
}
/**
* Commit-granular history: the rewindable state after the atomic commit that contains
* entry `at`, walking the fork chain for entries owned by ancestors.
*/
async docAsOf(conversationId: Id, at: Id, _ctx?: Context): Promise<JsonObject | undefined> {
const owner = this.entriesById.get(at)?.conversationId;
if (owner === undefined) return undefined;
const chain: Id[] = [];
let c: Conversation | undefined = this.conversationsById.get(conversationId);
while (c !== undefined && c.id !== owner) {
chain.push(c.id);
c = c.parent ? this.conversationsById.get(c.parent.conversationId) : undefined;
}
if (c === undefined) return undefined;
const seqAt = this.entrySeq.get(at) ?? 0;
const log = (this.rewindableLog.get(owner) ?? []).filter((r) => r.seq <= seqAt).map((r) => r.ops);
return log.length === 0 ? undefined : clone(this.fold(log));
}
protected fold(log: Op[][]): JsonObject {
let start = 0;
for (let i = log.length - 1; i >= 0; i--)
if (isBase(log[i]!)) {
start = i;
break;
}
let state: JsonObject = {};
for (let i = start; i < log.length; i++) state = applyImmutable(state, log[i]!) as JsonObject;
return state;
}
async truncate(ref: DocRef, _ctx?: Context) {
if (ref.doc !== "sticky") return;
const log = this.stickyLog.get(ref.conversationId);
if (log === undefined) return;
let start = 0;
for (let i = log.length - 1; i >= 0; i--)
if (isBase(log[i]!)) {
start = i;
break;
}
log.splice(0, start);
}
async close(_ctx?: Context) {
this.closed = true;
}
}
function clone<T>(value: T): T {
return JSON.parse(JSON.stringify(value)) as T;
}
function optionalClone<T>(value: T | undefined): T | undefined {
return value === undefined ? undefined : clone(value);
}

View file

@ -0,0 +1,486 @@
import type { Context } from "@earendil-works/chord";
import { withAbortSignal } from "@earendil-works/chord/context";
import type { Session, TxImpl } from "./session.ts";
import {
type AnyKind,
type Checkpoint,
type Completion,
type Id,
type Input,
InvocationToken,
type Invoker,
type JsonValue,
type Outcome,
type Runtime,
type Step,
type Task,
TaskContractFault,
} from "./types.ts";
interface Invocation {
task: Task;
mode: "run" | "abort";
controller: AbortController;
done: Promise<void>;
}
export interface SchedulerDeps {
session: Session;
kinds: ReadonlyMap<string, AnyKind>;
runtime(task: Task, invoker: Extract<Invoker, { type: "task" }>, ctx: Context): Runtime;
onReport(error: unknown): void;
ctx: Context;
}
/**
* The erased-kind adapter: the one place a concrete `Kind<…>` is called through `AnyKind`
* and its outputs are validated. A handler that throws or returns an invalid shape is a
* contract fault, recorded as `{ status: "faulted" }` — never a manufactured declared failure.
*/
type Handler = (task: Task, rt: Runtime, ctx: Context) => Promise<Step<Checkpoint, JsonValue, JsonValue>>;
function handlerFor(kind: AnyKind, phase: string | undefined): Handler {
const h = phase === undefined ? kind.initial : kind.phases[phase];
if (typeof h !== "function") throw new TaskContractFault(kind.name, `no handler for phase ${phase ?? "initial"}`);
return h as Handler;
}
function validateStep(kind: AnyKind, step: unknown): Step<Checkpoint, JsonValue, JsonValue> {
if (typeof step !== "object" || step === null)
throw new TaskContractFault(kind.name, "handler returned a non-object step");
if ("done" in step && typeof (step as { done: unknown }).done === "function")
return step as Step<Checkpoint, JsonValue, JsonValue>;
if ("next" in step) {
const n = (step as { next: unknown }).next;
if (typeof n === "function") return step as Step<Checkpoint, JsonValue, JsonValue>;
if (typeof n === "object" && n !== null && typeof (n as { phase?: unknown }).phase === "string") {
if (!((n as { phase: string }).phase in kind.phases))
throw new TaskContractFault(kind.name, `transition to unknown phase ${(n as { phase: string }).phase}`);
return step as Step<Checkpoint, JsonValue, JsonValue>;
}
}
throw new TaskContractFault(kind.name, "handler returned an invalid step");
}
function validateCompletion(kind: AnyKind, c: unknown): Completion<JsonValue, JsonValue> {
if (typeof c === "object" && c !== null) {
const s = (c as { status?: unknown }).status;
if (s === "completed" && "result" in c) return c as Completion<JsonValue, JsonValue>;
if (s === "failed" && "failure" in c) return c as Completion<JsonValue, JsonValue>;
}
throw new TaskContractFault(kind.name, "closure returned an invalid completion");
}
function validateCheckpoint(kind: AnyKind, cp: unknown): Checkpoint {
if (typeof cp !== "object" || cp === null || typeof (cp as { phase?: unknown }).phase !== "string")
throw new TaskContractFault(kind.name, "invalid checkpoint");
const phase = (cp as { phase: string }).phase;
if (!(phase in kind.phases)) throw new TaskContractFault(kind.name, `checkpoint names unknown phase ${phase}`);
if (kind.inflight?.includes(phase))
throw new TaskContractFault(
kind.name,
`transition into in-flight phase ${phase}; write it with rt.commit before the effect instead`,
);
JSON.stringify(cp);
return cp as Checkpoint;
}
export class Scheduler {
private readonly deps: SchedulerDeps;
private enabled = false;
private holds = 0;
private dirty = false;
private draining = false;
private readonly invocations = new Map<Id, Invocation>();
private readonly taskWaiters = new Map<Id, Set<(t: Task) => void>>();
private readonly inputWaiters = new Map<Id, Set<(i: Input) => void>>();
private readonly idleWaiters = new Set<{ conversationId: Id | undefined; resolve: () => void }>();
constructor(deps: SchedulerDeps) {
this.deps = deps;
deps.session.listeners.add((r) => {
for (const t of r.changes.tasks) {
const inv = this.invocations.get(t.id);
if (t.abort === true && inv?.mode === "run") inv.controller.abort();
}
for (const task of r.changes.tasks)
if (task.status === "terminal")
this.taskWaiters.get(task.id)?.forEach((waiter) => {
waiter(task);
});
for (const input of r.changes.inputs)
if (input.status === "done" || input.status === "unanswered")
this.inputWaiters.get(input.id)?.forEach((waiter) => {
waiter(input);
});
if (r.changes.tasks.length > 0) this.kick();
this.checkIdle();
});
}
resume() {
this.enabled = true;
this.kick();
}
stop() {
this.enabled = false;
}
hold(): () => void {
this.holds++;
let released = false;
return () => {
if (released) return;
released = true;
this.holds--;
if (this.holds === 0) this.kick();
};
}
kick() {
if (!this.enabled) return;
this.dirty = true;
if (this.holds === 0 && !this.draining) this.drain().catch((e) => this.deps.onReport(e));
}
private async drain() {
this.draining = true;
try {
while (this.dirty && this.enabled && this.holds === 0) {
this.dirty = false;
const reserved = await this.reserveEligible();
if (!this.enabled || this.holds > 0) {
this.dirty = true;
break;
}
for (const { task, mode } of reserved) this.dispatch(task, mode);
}
} finally {
this.draining = false;
this.checkIdle();
}
}
/** On the line: reserve every eligible task. */
private async reserveEligible(): Promise<{ task: Task; mode: "run" | "abort" }[]> {
const { session } = this.deps;
const out: { task: Task; mode: "run" | "abort" }[] = [];
await session.commit(
{ type: "kernel" },
async (tx) => {
for (const task of [...session.liveTasks.values()]) {
if (!this.deps.kinds.has(task.kind)) continue;
const inv = this.invocations.get(task.id);
if (task.abort === true) {
if (inv?.mode === "abort") continue;
if (inv?.mode === "run") continue; // winding down; dispatched after it returns
const running: Task = task.status === "pending" ? { ...task, status: "running" } : task;
if (running !== task) tx.setTask(running);
out.push({ task: running, mode: "abort" });
continue;
}
if (inv !== undefined) continue;
if (task.status === "pending") {
let ready = true;
for (const dep of task.after) {
const d = await tx.task(dep);
if (d === undefined || d.status !== "terminal") {
ready = false;
break;
}
}
if (!ready) continue;
const running: Task = { ...task, status: "running" };
tx.setTask(running);
out.push({ task: running, mode: "run" });
} else if (task.status === "running") {
out.push({ task, mode: "run" }); // found at open: dispatch into its current phase
}
}
},
this.deps.ctx,
);
return out;
}
private dispatch(task: Task, mode: "run" | "abort") {
const { session } = this.deps;
const kind = this.deps.kinds.get(task.kind);
if (kind === undefined) {
this.deps.onReport(new Error(`unknown kind ${task.kind} for task ${task.id}`));
return;
}
const controller = new AbortController();
const ctx = withAbortSignal(controller.signal, this.deps.ctx);
const done = (async () => {
try {
if (mode === "run") {
const closure = await this.runPhases(task, kind, ctx);
if (closure === undefined || controller.signal.aborted) return;
const lease = this.lease(task, kind, mode, ctx);
try {
await session.commit(
lease.invoker,
async (tx, lineCtx, control) => {
const current = session.liveTasks.get(task.id);
if (current === undefined || current.abort === true) return;
const completion = validateCompletion(
kind,
await (closure as (tx: TxImpl, task: Task, ctx: Context) => unknown)(tx, current, lineCtx),
);
const patched = (await tx.task(task.id)) ?? current;
control.setTask({ ...patched, status: "terminal", outcome: completion });
},
ctx,
{ closing: true },
);
} finally {
lease.token.revoke();
}
} else {
const handlerLease = this.lease(task, kind, mode, ctx);
let closure: (tx: TxImpl, task: Task, ctx: Context) => unknown;
try {
closure = await (
kind.abort as (
task: Task,
runtime: Runtime,
ctx: Context,
) => Promise<(tx: TxImpl, task: Task, ctx: Context) => unknown>
)(task, handlerLease.runtime, ctx);
} finally {
handlerLease.token.revoke();
}
const closureLease = this.lease(task, kind, mode, ctx);
try {
await session.commit(
closureLease.invoker,
async (tx, lineCtx, control) => {
const current = session.liveTasks.get(task.id);
if (current === undefined) return;
const result = (await closure(tx, current, lineCtx)) as JsonValue;
JSON.stringify(result);
const patched = (await tx.task(task.id)) ?? current;
control.setTask({ ...patched, status: "terminal", outcome: { status: "aborted", result } });
},
ctx,
{ closing: true },
);
} finally {
closureLease.token.revoke();
}
}
} catch (error) {
if (controller.signal.aborted) return;
// Contract fault or unexpected throw: the task ends `faulted`, never as a declared failure.
this.deps.onReport(error);
await session
.commit(
{ type: "kernel" },
async (tx) => {
const current = session.liveTasks.get(task.id);
if (current === undefined) return;
const outcome: Outcome = {
status: "faulted",
error: error instanceof TaskContractFault ? error.message : `${kind.name}: ${String(error)}`,
};
tx.setTask({ ...current, status: "terminal", outcome });
// A faulted turn task: its inputs would otherwise stay placed forever; the turn state is stale.
if (kind.turn) {
const inputs = (current.input as { inputs?: unknown }).inputs;
if (Array.isArray(inputs))
await tx.resolveInputs(
inputs.filter((i): i is Id => typeof i === "number"),
{ status: "unanswered", reason: "failed", detail: outcome.error },
);
tx.sticky(current.conversationId).turn = { tools: [] };
}
},
this.deps.ctx,
{ docs: [{ doc: "sticky", conversationId: task.conversationId }] },
)
.catch((e) => this.deps.onReport(e));
} finally {
this.invocations.delete(task.id);
const now = session.liveTasks.get(task.id);
if (now === undefined) await session.retire(task, this.deps.ctx).catch((e) => this.deps.onReport(e));
this.kick();
}
})();
this.invocations.set(task.id, { task, mode, controller, done });
}
private lease(task: Task, kind: AnyKind, mode: "run" | "abort", ctx: Context) {
const token = new InvocationToken(task.id, mode);
const invoker: Extract<Invoker, { type: "task" }> = {
type: "task",
token,
id: task.id,
conversationId: task.conversationId,
kind,
core:
task.kind.startsWith("pi.") &&
["pi.generation", "pi.tool", "pi.post_tools", "pi.collapse"].includes(task.kind),
mode,
};
return { token, invoker, runtime: this.deps.runtime(task, invoker, ctx) };
}
/** The phase loop. Each handler gets a capability lease revoked when that handler's promise returns. */
private async runPhases(task: Task, kind: AnyKind, ctx: Context): Promise<unknown> {
const { session } = this.deps;
let current = task;
for (;;) {
const cp = current.checkpoint;
const handler = handlerFor(kind, cp?.phase);
const handlerLease = this.lease(task, kind, "run", ctx);
let step: Step<Checkpoint, JsonValue, JsonValue>;
try {
step = validateStep(kind, await handler(current, handlerLease.runtime, ctx));
} finally {
handlerLease.token.revoke();
}
if ("done" in step) return step.done;
if (ctx.abortSignal?.aborted) return undefined;
const transitionLease = this.lease(task, kind, "run", ctx);
let outcome: { value: "discard" | "retry" | "terminal" | "advanced" };
try {
outcome = await session.commit(
transitionLease.invoker,
async (tx, lineCtx, control) => {
const live = session.liveTasks.get(task.id);
if (live === undefined || live.abort === true) return "discard" as const;
const next = typeof step.next === "function" ? await step.next(tx, live, lineCtx) : step.next;
if (next === "retry") return "retry" as const;
if (typeof next === "object" && next !== null && "status" in next) {
const patched = (await tx.task(task.id)) ?? live;
control.setTask({ ...patched, status: "terminal", outcome: validateCompletion(kind, next) });
return "terminal" as const;
}
const checkpoint = validateCheckpoint(kind, next);
tx.checkpoint(checkpoint);
current = { ...live, checkpoint };
return "advanced" as const;
},
ctx,
);
} finally {
transitionLease.token.revoke();
}
if (outcome.value === "retry") {
current = session.liveTasks.get(task.id) ?? current;
continue;
}
if (outcome.value !== "advanced") return undefined;
}
}
/** Mark, revoke, signal, join. The scheduler then reserves the fresh abort on its next drain. */
async abortTask(id: Id, ctx: Context): Promise<"marked" | "terminal"> {
const { session } = this.deps;
const r = await session.commit(
{ type: "kernel" },
async (tx) => {
const task = await tx.task(id);
if (task === undefined) throw new Error(`task ${id} not found`);
if (task.status === "terminal") return "terminal" as const;
if (task.abort !== true) tx.setTask({ ...task, abort: true });
return "marked" as const;
},
ctx,
);
if (r.value === "terminal") return "terminal";
const inv = this.invocations.get(id);
if (inv?.mode === "run") {
inv.controller.abort();
await inv.done;
}
this.kick();
return "marked";
}
// --- waiters: registered on the line, atomically with the state read -------------------
waitForTask(id: Id, ctx: Context): Promise<Task> {
return this.waiter(ctx, this.taskWaiters, id, (lineCtx) => {
const live = this.deps.session.liveTasks.get(id);
return live === undefined ? this.deps.session.storage.task(id, lineCtx) : Promise.resolve(undefined);
});
}
waitForInput(id: Id, ctx: Context): Promise<Input> {
return this.waiter(ctx, this.inputWaiters, id, async (lineCtx) => {
const i = await this.deps.session.storage.input(id, lineCtx);
return i !== undefined && (i.status === "done" || i.status === "unanswered") ? i : undefined;
});
}
/** Read state and install the waiter in one line operation; the wait itself happens off-line. */
private async waiter<T>(
ctx: Context,
table: Map<Id, Set<(v: T) => void>>,
id: Id,
terminal: (ctx: Context) => Promise<T | undefined>,
): Promise<T> {
ctx.abortSignal?.throwIfAborted();
const r = await this.deps.session.onLine(async (lineCtx): Promise<{ now: T } | { pending: Promise<T> }> => {
const now = await terminal(lineCtx);
if (now !== undefined) return { now };
return {
pending: new Promise<T>((resolve, reject) => {
const set = table.get(id) ?? table.set(id, new Set()).get(id)!;
const w = (v: T) => {
set.delete(w);
ctx.abortSignal?.removeEventListener("abort", onAbort);
resolve(v);
};
const onAbort = () => {
set.delete(w);
reject(ctx.abortSignal?.reason ?? new Error("aborted"));
};
set.add(w);
ctx.abortSignal?.addEventListener("abort", onAbort, { once: true });
}),
};
}, ctx);
return "now" in r ? r.now : r.pending;
}
async waitForIdle(conversationId: Id | undefined, ctx: Context): Promise<void> {
ctx.abortSignal?.throwIfAborted();
const r = await this.deps.session.onLine((): { pending?: Promise<void> } => {
if (this.isIdle(conversationId)) return {};
return {
pending: new Promise<void>((resolve, reject) => {
const w = {
conversationId,
resolve: () => {
this.idleWaiters.delete(w);
ctx.abortSignal?.removeEventListener("abort", onAbort);
resolve();
},
};
const onAbort = () => {
this.idleWaiters.delete(w);
reject(ctx.abortSignal?.reason ?? new Error("aborted"));
};
this.idleWaiters.add(w);
ctx.abortSignal?.addEventListener("abort", onAbort, { once: true });
}),
};
}, ctx);
return r.pending;
}
private isIdle(conversationId: Id | undefined): boolean {
for (const t of this.deps.session.liveTasks.values())
if ((conversationId === undefined || t.conversationId === conversationId) && !t.background) return false;
return true;
}
private checkIdle() {
for (const w of [...this.idleWaiters]) if (this.isIdle(w.conversationId) && !this.draining) w.resolve();
}
/** Signal every invocation and wait for them; nothing is written. */
async joinAll() {
this.enabled = false;
for (const inv of this.invocations.values()) inv.controller.abort();
await Promise.allSettled([...this.invocations.values()].map((invocation) => invocation.done));
}
quiescent(): boolean {
return this.invocations.size === 0;
}
get liveInvocations() {
return this.invocations.size;
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,376 @@
import type { Context } from "@earendil-works/chord";
import type {
AnyToolDeclaration,
ContextEdit,
CoreTx,
Entry,
HookInfo,
HookResult,
Id,
JsonValue,
RewindableState,
Runtime,
Stored,
SystemMessage,
} from "./types.ts";
// ---------------------------------------------------------------------------
// Sections (pico §12.1). A section is a stable key plus a pure renderer. Only
// keys, payloads and rendered text are stored; never renderers.
// ---------------------------------------------------------------------------
export interface SystemSection<T extends JsonValue = JsonValue> {
readonly key: string;
render(value: T): string;
}
export function defineSystemSection<T extends JsonValue>(definition: {
key: string;
render(value: T): string;
}): SystemSection<T> {
return Object.freeze(definition);
}
export type EnvironmentInfo = { cwd: string };
export type SkillInfo = { name: string; description: string };
export const systemSections = {
identity: defineSystemSection<string>({ key: "identity", render: (v) => v }),
environment: defineSystemSection<EnvironmentInfo>({
key: "environment",
render: (v) => `Working directory: ${v.cwd}`,
}),
skills: defineSystemSection<SkillInfo[]>({
key: "skills",
render: (v) => v.map((s) => `- ${s.name}: ${s.description}`).join("\n"),
}),
} as const;
/** Section seed for a new conversation (§8.1): set with a checked pair, or remove an inherited key. */
export type SectionSeed = { key: string; value: JsonValue } | { key: string; remove: true };
export const sectionSeed = <T extends JsonValue>(section: SystemSection<T>, value: T): SectionSeed => ({
key: section.key,
value,
});
export const removeSection = (key: string): SectionSeed => ({ key, remove: true });
// ---------------------------------------------------------------------------
// The managed entry (§12.3)
// ---------------------------------------------------------------------------
export type SectionRecord =
| { key: string; action: "set"; value: JsonValue; rendered: string }
| { key: string; action: "remove" };
export type SystemEntryData = { baseline?: true; sections: SectionRecord[] };
/** A metadata-only delta has `model: []`. */
export type SystemEntry = Entry & { kind: "pi.system"; model: [] | [Stored<SystemMessage>]; data: SystemEntryData };
// ---------------------------------------------------------------------------
// Canonical state (§12.4): fold the fork-visible managed entries from the
// newest baseline forward. Uses the kind scan; never depends on heads or edits.
// ---------------------------------------------------------------------------
type SectionState = { value: JsonValue; rendered: string };
export type Canonical = Map<string, SectionState>; // insertion order is canonical order
export async function foldCanonical(
reads: Pick<CoreTx, "scanEntries">,
conversationId: Id,
): Promise<{ canonical: Canonical; newestManaged: Id | null; newestBaseline: Id | null }> {
const managed: SystemEntry[] = [];
let before: Id | undefined;
let baseline: Id | null = null;
scan: for (;;) {
const page = await reads.scanEntries({
conversationId,
kind: "pi.system",
limit: 64,
...(before === undefined ? {} : { before }),
});
for (const e of page) {
managed.push(e as SystemEntry);
if ((e as SystemEntry).data.baseline) {
baseline = e.id;
break scan;
}
}
if (page.length < 64) break;
before = page[page.length - 1]!.id;
}
managed.reverse();
const canonical: Canonical = new Map();
for (const e of managed)
for (const r of e.data.sections)
r.action === "remove"
? canonical.delete(r.key)
: canonical.set(r.key, { value: r.value, rendered: r.rendered });
return { canonical, newestManaged: managed[managed.length - 1]?.id ?? null, newestBaseline: baseline };
}
// ---------------------------------------------------------------------------
// Draft (§12.2): what `systemInstructions` handlers edit.
// ---------------------------------------------------------------------------
export interface SystemSectionDraft {
get<T extends JsonValue>(section: SystemSection<T>): T | undefined; // owned copy
set<T extends JsonValue>(section: SystemSection<T>, value: T): void; // existing key keeps position; new key appends
delete(section: { key: string }): void;
wrap<T extends JsonValue>(section: SystemSection<T>, transform: (rendered: string) => string): void; // preparation-local
}
class Draft implements SystemSectionDraft {
values = new Map<string, JsonValue>();
wrappers = new Map<string, ((r: string) => string)[]>();
touched = new Set<string>();
constructor(seed: Canonical) {
for (const [k, s] of seed) this.values.set(k, s.value);
}
get<T extends JsonValue>(section: SystemSection<T>) {
const v = this.values.get(section.key);
return v === undefined ? undefined : (structuredClone(v) as T);
}
set<T extends JsonValue>(section: SystemSection<T>, value: T) {
this.values.set(section.key, value);
this.touched.add(section.key);
}
delete(section: { key: string }) {
this.values.delete(section.key);
this.wrappers.delete(section.key);
this.touched.add(section.key);
}
wrap<T extends JsonValue>(section: SystemSection<T>, transform: (r: string) => string) {
(this.wrappers.get(section.key) ?? this.wrappers.set(section.key, []).get(section.key)!).push(transform);
this.touched.add(section.key);
}
snapshot() {
return {
values: new Map(this.values),
wrappers: new Map([...this.wrappers].map(([k, v]) => [k, [...v]])),
touched: new Set(this.touched),
};
}
restore(s: ReturnType<Draft["snapshot"]>) {
this.values = s.values;
this.wrappers = s.wrappers;
this.touched = s.touched;
}
}
/** Render touched sections (wrappers apply after rendering, in registration order); untouched keep their stored text. */
function freeze(draft: Draft, canonical: Canonical, registry: ReadonlyMap<string, SystemSection>): Canonical {
const desired: Canonical = new Map();
for (const [key, value] of draft.values) {
if (!draft.touched.has(key)) {
const prev = canonical.get(key);
if (prev) desired.set(key, prev);
continue;
}
const def = registry.get(key);
if (def === undefined) continue; // unregistered: cannot re-render; treated as removed
let rendered = (def as SystemSection).render(value);
for (const w of draft.wrappers.get(key) ?? []) rendered = w(rendered);
desired.set(key, { value, rendered });
}
return desired;
}
// ---------------------------------------------------------------------------
// Preparation (§12.5)
// ---------------------------------------------------------------------------
export interface SectionRegistry {
readonly map: ReadonlyMap<string, SystemSection>;
readonly revision: number;
}
export interface ToolRegistry {
readonly map: ReadonlyMap<string, AnyToolDeclaration>;
readonly revision: number;
}
/** Snapshot S. Compared field by field before the `prepared` commit. */
export interface PreparationSnapshot {
newestManaged: Id | null;
newestBaseline: Id | null;
newestHead: Id | null;
settings: { model?: JsonValue; thinkingLevel: JsonValue; selectedTools: string[]; profile: JsonValue };
sectionsRev: number;
toolsRev: number;
}
export const sameSnapshot = (a: PreparationSnapshot, b: PreparationSnapshot) => JSON.stringify(a) === JSON.stringify(b);
export async function takeSnapshot(
tx: Pick<CoreTx, "scanEntries" | "newestEntry" | "snapshot" | "conversation">,
conversationId: Id,
sections: SectionRegistry,
tools: ToolRegistry,
): Promise<{ snapshot: PreparationSnapshot; canonical: Canonical; seed: readonly SectionSeed[] | undefined }> {
const { canonical, newestManaged, newestBaseline } = await foldCanonical(tx, conversationId);
const head = await tx.newestEntry(conversationId, { withHead: true });
const r = tx.snapshot({ doc: "rewindable", conversationId }); // a plain copy: the snapshot outlives this transaction
const conv = await tx.conversation(conversationId);
return {
snapshot: {
newestManaged,
newestBaseline,
newestHead: head?.id ?? null,
settings: {
...(r.model === undefined ? {} : { model: r.model }),
thinkingLevel: r.thinkingLevel,
selectedTools: [...r.selectedTools],
profile: r.profile,
},
sectionsRev: sections.revision,
toolsRev: tools.revision,
},
canonical,
seed: newestManaged === null ? conv?.sections : undefined, // §8.1: the seed applies while there is no local managed entry
};
}
export interface SystemInstructionsHooks {
/** Edit the draft; optionally override the tool loadout (last override wins). A throwing handler's edits are rolled back. */
systemInstructions(
input: {
sections: SystemSectionDraft;
config: PreparationSnapshot["settings"];
tools: readonly AnyToolDeclaration[];
},
info: HookInfo,
ctx: Context,
): HookResult<{ tools?: readonly AnyToolDeclaration[] }>;
}
/** Off the line: seed the draft, run handlers, freeze. */
export async function prepareDraft(
rt: Pick<Runtime<SystemInstructionsHooks>, "hooks" | "tools">,
sections: SectionRegistry,
canonical: Canonical,
seed: readonly SectionSeed[] | undefined,
settings: PreparationSnapshot["settings"],
onReport: (m: string) => void,
ctx: Context,
): Promise<{ desired: Canonical; tools: readonly AnyToolDeclaration[] }> {
const draft = new Draft(canonical);
for (const s of seed ?? [])
"remove" in s ? draft.delete({ key: s.key }) : draft.set({ key: s.key, render: () => "" }, s.value);
const defaultTools: AnyToolDeclaration[] = [];
for (const name of settings.selectedTools) {
const t = rt.tools.get(name);
t ? defaultTools.push(t) : onReport(`selected tool ${name} is not registered`);
}
let tools: readonly AnyToolDeclaration[] = defaultTools;
for (const binding of rt.hooks.handlers()) {
const before = draft.snapshot();
try {
const out = await binding.handlers.systemInstructions?.(
{ sections: draft, config: settings, tools: defaultTools },
binding.api,
ctx,
);
if (out?.tools) tools = out.tools;
} catch (error) {
if (ctx.abortSignal?.aborted) throw error;
draft.restore(before); // collect / skip: this handler's edits are rolled back
onReport(String(error));
}
}
return { desired: freeze(draft, canonical, sections.map), tools };
}
/** On the line, in the `prepared` commit: diff desired against canonical; baseline if a head intervened. */
export async function planManagedEntry(
tx: Pick<CoreTx, "context">,
conversationId: Id,
snapshot: PreparationSnapshot,
canonical: Canonical,
desired: Canonical,
tools: readonly AnyToolDeclaration[],
now: number,
): Promise<{ data: SystemEntryData; model: [] | [Stored<SystemMessage>]; edits?: ContextEdit[] } | undefined> {
// §12.4: no managed history at all → first preparation appends a full baseline; and a usable
// baseline must follow the newest head.
const needBaseline =
snapshot.newestManaged === null ||
(snapshot.newestHead !== null &&
(snapshot.newestBaseline === null || snapshot.newestHead > snapshot.newestBaseline));
const toolOf = (t: AnyToolDeclaration) => ({
name: t.name,
description: t.description,
parameters: structuredClone(t.parameters) as Stored<SystemMessage>["toolsAdded"] extends (infer X)[] | undefined
? X extends { parameters: infer PP }
? PP
: never
: never,
});
// Previous effective tools: those declared by the newest managed entry chain (we recompute from canonical's owner: the tail message).
const { messages } = await tx.context(conversationId);
const previous = new Map<string, { name: string }>();
for (const m of messages)
if (m.role === "system") {
for (const t of (m as SystemMessage).toolsRemoved ?? []) previous.delete(t.name);
for (const t of (m as SystemMessage).toolsAdded ?? []) previous.set(t.name, t);
}
if (needBaseline) {
const sections: SectionRecord[] = [...desired].map(([key, s]) => ({
key,
action: "set",
value: s.value,
rendered: s.rendered,
}));
const content = sections.map((s) => (s.action === "set" ? `## ${s.key}\n${s.rendered}` : "")).join("\n\n");
const { entries } = await tx.context(conversationId);
const edits: ContextEdit[] = entries
.filter((e: Entry) => e.kind === "pi.system" && e.id > (snapshot.newestHead ?? 0))
.map((e: Entry) => ({ target: e.id, action: "omit" as const }));
const message = {
role: "system",
content,
toolsAdded: tools.map(toolOf),
timestamp: now,
} as Stored<SystemMessage>;
return { data: { baseline: true, sections }, model: [message], ...(edits.length ? { edits } : {}) };
}
const changed: SectionRecord[] = [];
for (const [key, s] of desired) {
const prev = canonical.get(key);
if (!prev || prev.rendered !== s.rendered || JSON.stringify(prev.value) !== JSON.stringify(s.value))
changed.push({ key, action: "set", value: s.value, rendered: s.rendered });
}
for (const key of canonical.keys()) if (!desired.has(key)) changed.push({ key, action: "remove" });
const added = tools.filter((t) => !previous.has(t.name));
const removed = [...previous.values()].filter((p) => !tools.some((t) => t.name === p.name));
if (changed.length === 0 && added.length === 0 && removed.length === 0) return undefined;
const renderChanged = changed.some((s) => s.action === "remove" || canonical.get(s.key)?.rendered !== s.rendered);
if (!renderChanged && added.length === 0 && removed.length === 0) return { data: { sections: changed }, model: [] }; // metadata-only delta
const content = changed
.map((s) =>
s.action === "set"
? `The ${s.key} section now reads:\n${s.rendered}`
: `The ${s.key} section no longer applies.`,
)
.join("\n\n");
const message = {
role: "system",
content,
...(added.length ? { toolsAdded: added.map(toolOf) } : {}),
...(removed.length ? { toolsRemoved: removed } : {}),
timestamp: now,
} as Stored<SystemMessage>;
return { data: { sections: changed }, model: [message] };
}
/** Tools effective in a projected request: fold toolsAdded/toolsRemoved across its SystemMessages. */
export function effectiveTools(messages: readonly { role: string }[]): { name: string }[] {
const tools = new Map<string, { name: string }>();
for (const m of messages)
if (m.role === "system") {
for (const t of (m as SystemMessage).toolsRemoved ?? []) tools.delete(t.name);
for (const t of (m as SystemMessage).toolsAdded ?? []) tools.set(t.name, t);
}
return [...tools.values()];
}
export type { RewindableState, Runtime };

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,477 @@
import { applyImmutable, type Op, type Tracker, track } from "@earendil-works/chord/delta";
import type { CommitResult, Session } from "./session.ts";
import type {
AnyKind,
Conversation,
ConversationView,
DocRef,
Envelope,
GenerationStatus,
Id,
JsonObject,
JsonValue,
RewindableState,
StickyState,
Task,
ToolSlot,
TurnView,
ViewEvent,
} from "./types.ts";
export const WATCH_CAPACITY = 256;
export interface Watch {
readonly view: ConversationView;
readonly revision: number;
readonly closed: boolean;
start(listener: (envelope: Envelope) => void): void;
stop(): void;
}
export function applyEnvelope(view: ConversationView, envelope: Envelope): ConversationView {
return applyImmutable(view, envelope.ops) as ConversationView;
}
type Listener = (envelope: Envelope) => void;
type ViewRecord = {
readonly conversationId: Id;
readonly tracker: Tracker<ConversationView>;
readonly watchers: Set<WatchImpl>;
revision: number;
};
type Delivery = { envelope: Envelope; watchers: WatchImpl[] };
export class ViewManager {
private readonly records = new Map<Id, ViewRecord>();
private readonly deliveries: Delivery[] = [];
private readonly session: Session;
private readonly onReport: (error: unknown) => void;
constructor(session: Session, onReport: (error: unknown) => void) {
this.session = session;
this.onReport = onReport;
}
watch(conversation: Conversation, entries: EntryList): Watch {
let record = this.records.get(conversation.id);
if (record === undefined) {
const tracker = track(this.build(conversation, entries));
tracker.flush();
record = { conversationId: conversation.id, tracker, watchers: new Set(), revision: 0 };
this.records.set(conversation.id, record);
}
const watcher = new WatchImpl(structuredClone(record.tracker.target), record.revision, this.onReport, () => {
record!.watchers.delete(watcher);
if (record!.watchers.size === 0 && this.records.get(conversation.id) === record)
this.records.delete(conversation.id);
});
record.watchers.add(watcher);
return watcher;
}
/** Runs on the Session line after persistence and in-memory indexes update. */
update(result: CommitResult<unknown>): void {
for (const record of [...this.records.values()]) {
const conversation = this.session.conversationRecords.get(record.conversationId);
if (conversation === undefined) continue;
try {
const changes = result.changes;
const appended = changes.entries.filter((entry) => entry.conversationId === record.conversationId);
const documentChanges = changes.docs.filter(
(change) => change.ref.doc === "session" || change.ref.conversationId === record.conversationId,
);
const taskChanged = changes.tasks.some((task) => task.conversationId === record.conversationId);
const events = changes.events
.filter((scoped) => scoped.conversationId === record.conversationId)
.map((scoped) => scoped.event);
if (appended.length === 0 && documentChanges.length === 0 && !taskChanged && events.length === 0) continue;
const entryOps = applyEntries(record.tracker, appended);
const state = record.tracker.state;
const rewindableChanged = documentChanges.some(
(change) => change.ref.doc === "rewindable" && change.ref.conversationId === record.conversationId,
);
const stickyChanged = documentChanges.some(
(change) => change.ref.doc === "sticky" && change.ref.conversationId === record.conversationId,
);
const configChanged = events.some((event) => event.type === "config.changed");
const pluginChanged = documentChanges.some((change) => touchesKey(change.ops, "plugins"));
const rewindable =
rewindableChanged || configChanged || pluginChanged
? this.document<RewindableState>({ doc: "rewindable", conversationId: record.conversationId })
: undefined;
const sticky =
stickyChanged || taskChanged || configChanged || pluginChanged
? this.document<StickyState>({ doc: "sticky", conversationId: record.conversationId })
: undefined;
if (configChanged) syncRecord(state.config, this.config(rewindable!, sticky!));
if (stickyChanged) syncArray(state.inbox, structuredClone(sticky!.inbox));
if (stickyChanged || taskChanged) {
syncOptional(state, "turn", this.turn(record.conversationId, sticky!).turn);
syncOptional(state, "compaction", this.compaction(record.conversationId).compaction);
syncRecord(state.tasks, this.tasks(record.conversationId, sticky!));
}
if (pluginChanged) {
const session = this.document<JsonObject>({ doc: "session" });
syncRecord(state.plugins, this.plugins(rewindable!, sticky!, session));
}
const ops = [...entryOps, ...record.tracker.flush()];
if (ops.length === 0 && events.length === 0) continue;
const envelope = Object.freeze({
revision: ++record.revision,
ops: Object.freeze(ops) as unknown as Op[],
events: Object.freeze(events) as unknown as ViewEvent[],
}) satisfies Envelope;
this.deliveries.push({ envelope, watchers: [...record.watchers] });
} catch (error) {
this.records.delete(record.conversationId);
for (const watcher of [...record.watchers]) watcher.fail(error);
}
}
}
/** Runs after the Session line; listeners are synchronous and ordered. */
deliver(): void {
for (const delivery of this.deliveries.splice(0)) {
for (const watcher of delivery.watchers) watcher.accept(delivery.envelope);
}
}
close(): void {
const records = [...this.records.values()];
this.records.clear();
this.deliveries.length = 0;
for (const record of records) for (const watcher of [...record.watchers]) watcher.stop();
}
private build(conversation: Conversation, entries: EntryList): ConversationView {
const rewindable = this.document<RewindableState>({ doc: "rewindable", conversationId: conversation.id });
const sticky = this.document<StickyState>({ doc: "sticky", conversationId: conversation.id });
const session = this.document<JsonObject>({ doc: "session" });
const { sections: _sections, ...publicConversation } = conversation;
return {
conversation: structuredClone(publicConversation),
entries: structuredClone(entries),
config: this.config(rewindable, sticky),
inbox: structuredClone(sticky.inbox),
...this.turn(conversation.id, sticky),
...this.compaction(conversation.id),
tasks: this.tasks(conversation.id, sticky),
plugins: this.plugins(rewindable, sticky, session),
};
}
private document<T>(ref: DocRef): T {
const value = this.session.loadedDocument(ref);
if (value === undefined) throw new Error(`view document ${ref.doc} is not loaded`);
return value as T;
}
private config(rewindable: RewindableState, sticky: StickyState): ConversationView["config"] {
const out: ConversationView["config"] = {};
for (const [key, doc] of this.session.defaults.route) {
const source = doc === "rewindable" ? rewindable : sticky;
const value = key in source ? source[key] : this.session.defaults[doc][key];
if (value !== undefined) out[key] = structuredClone(value);
}
return out;
}
private turn(conversationId: Id, sticky: StickyState): { turn?: TurnView } {
const live = [...this.session.liveTasks.values()].filter((task) => task.conversationId === conversationId);
const turnTasks = live.filter((task) => this.session.kinds.get(task.kind)?.turn === true);
if (turnTasks.length === 0) return {};
const generation = turnTasks.find((task) => task.kind === "pi.generation");
const postTools = turnTasks.find((task) => task.kind === "pi.post_tools");
const inputTask = generation ?? postTools;
const inputs = inputTask === undefined ? [] : inputIds(inputTask);
const turn: TurnView = {
inputs,
...(generation === undefined
? {}
: { generation: generationStatus(generation, sticky.turn.message !== undefined) }),
...(sticky.turn.message === undefined ? {} : { message: structuredClone(sticky.turn.message) }),
tools: sticky.turn.tools.map(stripPrivateToolState),
};
return { turn };
}
private compaction(conversationId: Id): { compaction?: ConversationView["compaction"] } {
const task = [...this.session.liveTasks.values()].find(
(candidate) => candidate.conversationId === conversationId && candidate.kind === "pi.collapse",
);
if (task === undefined) return {};
const input = task.input as { reason: "threshold" | "manual" | "overflow" };
const checkpoint = task.checkpoint as { phase?: string; attempt?: number; untilMs?: number } | undefined;
return {
compaction: {
taskId: task.id,
reason: input.reason,
stage: checkpoint?.phase === "retrying" ? "retrying" : "summarizing",
attempt: checkpoint?.attempt ?? 1,
...(checkpoint?.phase === "retrying" && checkpoint.untilMs !== undefined
? { retryAt: checkpoint.untilMs }
: {}),
},
};
}
private tasks(conversationId: Id, sticky: StickyState): ConversationView["tasks"] {
const out: ConversationView["tasks"] = {};
for (const task of this.session.liveTasks.values()) {
const kind = this.session.kinds.get(task.kind);
if (
task.conversationId !== conversationId ||
kind === undefined ||
kind.turn === true ||
task.kind === "pi.collapse"
)
continue;
const slot = sticky.tasks[task.id];
out[task.id] = {
kind: task.kind,
...(task.background ? { background: true as const } : {}),
...(task.abort ? { marked: true as const } : {}),
status: describe(kind, task, slot),
};
}
return out;
}
private plugins(rewindable: RewindableState, sticky: StickyState, session: JsonObject): ConversationView["plugins"] {
const out: ConversationView["plugins"] = {};
const sessionPlugins = (session.plugins ?? {}) as { [key: string]: JsonObject };
for (const [id, registration] of this.session.namespaces) {
if (registration.project === undefined) continue;
const merged: JsonObject = {
...registration.defaults.rewindable,
...registration.defaults.sticky,
...registration.defaults.session,
...(rewindable.plugins[id] ?? {}),
...(sticky.plugins[id] ?? {}),
...(sessionPlugins[id] ?? {}),
};
out[id] = strictJson(registration.project(structuredClone(merged)), `namespace ${id} view`);
}
return out;
}
}
type EntryList = ConversationView["entries"];
class WatchImpl implements Watch {
readonly view: ConversationView;
private readonly initialRevision: number;
private readonly onReport: (error: unknown) => void;
private readonly onStop: () => void;
private listener: Listener | undefined;
private readonly buffer: Envelope[] = [];
private stopped = false;
constructor(view: ConversationView, revision: number, onReport: (error: unknown) => void, onStop: () => void) {
this.view = view;
this.initialRevision = revision;
this.onReport = onReport;
this.onStop = onStop;
}
get revision(): number {
return this.initialRevision;
}
get closed(): boolean {
return this.stopped;
}
start(listener: Listener): void {
if (this.listener !== undefined || this.stopped) return;
this.listener = listener;
for (const envelope of this.buffer.splice(0)) {
if (this.stopped) break;
this.deliver(envelope);
}
}
stop(): void {
if (this.stopped) return;
this.stopped = true;
this.buffer.length = 0;
this.onStop();
}
accept(envelope: Envelope): void {
if (this.stopped) return;
if (this.listener === undefined) {
if (this.buffer.length >= WATCH_CAPACITY) {
this.stop();
this.report(new Error(`watch capacity ${WATCH_CAPACITY} exceeded before start()`));
return;
}
this.buffer.push(envelope);
return;
}
this.deliver(envelope);
}
private deliver(envelope: Envelope): void {
try {
this.listener!(envelope);
} catch (error) {
this.fail(error);
}
}
fail(error: unknown): void {
this.stop();
this.report(error);
}
private report(error: unknown): void {
try {
this.onReport(error);
} catch {}
}
}
function applyEntries(tracker: Tracker<ConversationView>, appended: EntryList): Op[] {
const ops: Op[] = [];
const entries = tracker.state.entries;
for (const entry of appended) {
if (entry.head !== undefined) {
const retained = entries.findIndex((candidate) => candidate.id >= entry.head!);
const remove = retained < 0 ? entries.length : retained;
if (remove > 0) {
ops.push(...tracker.flush());
entries.splice(0, remove);
tracker.flush();
ops.push(["p", ["entries"], 0, remove, []]);
}
}
entries.push(structuredClone(entry));
}
return ops;
}
function inputIds(task: Task): Id[] {
const inputs = (task.input as { inputs?: JsonValue }).inputs;
return Array.isArray(inputs) ? inputs.filter((value): value is Id => typeof value === "number") : [];
}
function generationStatus(task: Task, streaming: boolean): GenerationStatus {
if (task.status === "pending" && task.after.length > 0) return { stage: "waiting", on: "compaction" };
const checkpoint = task.checkpoint as
| {
phase?: string;
attempt?: number;
untilMs?: number;
lastError?: string;
pollAt?: number;
}
| undefined;
if (checkpoint?.phase === "requesting")
return { stage: streaming ? "streaming" : "requesting", attempt: checkpoint.attempt ?? 1 };
if (checkpoint?.phase === "retrying")
return {
stage: "retrying",
attempt: checkpoint.attempt ?? 1,
retryAt: checkpoint.untilMs ?? 0,
lastError: checkpoint.lastError ?? "",
};
if (checkpoint?.phase === "deferred")
return { stage: "deferred", attempt: checkpoint.attempt ?? 1, pollAt: checkpoint.pollAt ?? 0 };
return { stage: "preparing" };
}
function stripPrivateToolState(slot: ToolSlot): Omit<ToolSlot, "memos"> {
const { memos: _memos, ...view } = structuredClone(slot);
return view;
}
function describe(kind: AnyKind, task: Task, slot: JsonObject | undefined): JsonValue {
if (typeof kind.describe !== "function")
return { phase: task.checkpoint?.phase ?? (task.status === "pending" ? "pending" : "running") };
const publicSlot = slot === undefined ? undefined : structuredClone(slot);
if (publicSlot !== undefined) delete publicSlot.memos;
const value = (kind.describe as (task: Task & { readonly slot?: Readonly<JsonObject> }) => JsonValue)({
...task,
...(publicSlot === undefined ? {} : { slot: publicSlot }),
});
return strictJson(value, `task kind ${task.kind} describe`);
}
function strictJson(value: JsonValue, what: string): JsonValue {
const encoded = JSON.stringify(value);
if (encoded === undefined) throw new Error(`${what} returned a non-JSON value`);
return JSON.parse(encoded) as JsonValue;
}
function touchesKey(ops: readonly Op[], key: string): boolean {
return ops.some((op) => op[0] === "r" || op[1][0] === key);
}
function syncOptional(target: object, key: string, next: unknown): void {
const record = target as Record<string, unknown>;
if (next === undefined) {
delete record[key];
return;
}
const current = record[key];
if (same(current, next)) return;
if (isRecord(current) && isRecord(next)) syncRecord(current, next);
else record[key] = structuredClone(next);
}
function syncRecord(target: Record<string, unknown>, next: Record<string, unknown>): void {
for (const key of Object.keys(target)) if (!(key in next)) delete target[key];
for (const [key, value] of Object.entries(next)) {
const current = target[key];
if (same(current, value)) continue;
if (Array.isArray(current) && Array.isArray(value)) {
syncArray(current, value);
continue;
}
if (isRecord(current) && isRecord(value)) {
syncRecord(current, value);
continue;
}
target[key] = structuredClone(value);
}
}
function syncArray(target: unknown[], next: unknown[]): void {
let prefix = 0;
while (prefix < target.length && prefix < next.length) {
if (same(target[prefix], next[prefix])) {
prefix++;
continue;
}
const current = target[prefix];
const replacement = next[prefix];
if (!isRecord(current) || !isRecord(replacement) || !sameArrayItem(current, replacement)) break;
syncRecord(current, replacement);
prefix++;
}
if (prefix === target.length && prefix === next.length) return;
if (prefix === target.length) {
target.push(...structuredClone(next.slice(prefix)));
return;
}
target.splice(prefix, target.length - prefix, ...structuredClone(next.slice(prefix)));
}
function sameArrayItem(current: Record<string, unknown>, next: Record<string, unknown>): boolean {
if (typeof current.id === "number" || typeof next.id === "number") return current.id === next.id;
if (typeof current.callId === "string" || typeof next.callId === "string") return current.callId === next.callId;
if (typeof current.type === "string" || typeof next.type === "string") return current.type === next.type;
return false;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function same(a: unknown, b: unknown): boolean {
if (Object.is(a, b)) return true;
return JSON.stringify(a) === JSON.stringify(b);
}

View file

@ -0,0 +1,256 @@
import assert from "node:assert/strict";
import { appendFileSync, readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { Type } from "typebox";
import { onTestFinished, test } from "vitest";
import { kinds } from "../../../src/harness/pico3/harness.ts";
import { JsonlStorage } from "../../../src/harness/pico3/jsonl.ts";
import { defineTask, type Namespace, type ToolDeclaration } from "../../../src/harness/pico3/types.ts";
import { ctx, echoScript, fake, Gate, open, untilPhase } from "./helpers.ts";
/** Drop the last `n` records of a file: simulates a crash after the other files of a commit were fsync'd. */
const chop = (file: string, n: number) => {
const lines = readFileSync(file, "utf8").split("\n").filter(Boolean);
writeFileSync(
file,
lines
.slice(0, lines.length - n)
.map((l) => `${l}\n`)
.join(""),
);
};
const last = (file: string) =>
JSON.parse(readFileSync(file, "utf8").trim().split("\n").pop()!) as {
seq: number;
refs?: string[];
writes: { type: string }[];
};
test("multi-file commit torn before main → the sidecar half is discarded on replay (no inbox item without its Input)", async () => {
const gate = new Gate();
let env = await open({ backend: "jsonl", models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
await gate.arrivals(1);
const f = await env.root.send({ content: "F" }, ctx); // busy → queued: one commit writing sticky (inbox) + main (Input)
await env.crash();
const sticky = last(join(env.dir!, "sticky-1.jsonl"));
assert.equal(last(join(env.dir!, "main.jsonl")).seq, sticky.seq); // main half landed…
chop(join(env.dir!, "main.jsonl"), 1); // …until we tear it off: sticky fsync'd, main not
env = await open({ dir: env.dir, backend: "jsonl", models: fake({ respond: echoScript }) });
assert.equal(await env.input(f.id), undefined); // the Input never existed
assert.deepEqual((await env.root.sticky(ctx)).inbox, []); // and the inbox does not reference it
await env.root.waitForIdle(ctx);
assert.equal((await env.input(a.id))?.status, "done"); // the earlier commit is intact
});
test("checkpoint before an effect: marker landed → durable, the effect does not rerun; marker torn → dropped, the effect reruns (it never started)", async () => {
const gate = new Gate();
const t: ToolDeclaration & { calls: number } = {
calls: 0,
name: "x",
description: "",
parameters: Type.Object({ v: Type.String() }),
replay: "unsafe",
async execute(_a, _api, c) {
t.calls++;
await gate.wait(c);
return { content: [{ type: "text", text: "ran" }] };
},
};
let env = await open({ backend: "jsonl", tools: [t] });
onTestFinished(() => env.close());
await env.root.send({ content: "tool:x" }, ctx);
const task = await untilPhase(env, "pi.tool", "started");
await gate.arrivals(1);
await env.crash();
const rec = last(join(env.dir!, `task-${task.id}.jsonl`));
const marker = last(join(env.dir!, "main.jsonl"));
assert.equal(marker.seq, rec.seq); // the marker is the publication point
assert.deepEqual([...marker.refs!].sort(), [`sticky-1.jsonl`, `task-${task.id}.jsonl`].sort());
assert.deepEqual(marker.writes, []);
// (a) intact: replay restores `started`; replay "unsafe" → synthetic interrupted result, NOT a second run
const t2 = { ...t, calls: 0 };
env = await open({ dir: env.dir, backend: "jsonl", tools: [t2] });
await env.root.waitForIdle(ctx);
assert.equal(t2.calls, 0);
assert.match(JSON.stringify((await env.entries()).find((e) => e.kind === "pi.tool_result")!.model), /interrupted/);
});
test("checkpoint before an effect, marker torn: the checkpoint is dropped and the phase runs from initial; the effect never ran, so this is safe", async () => {
const gate = new Gate();
const t: ToolDeclaration & { calls: number } = {
calls: 0,
name: "x",
description: "",
parameters: Type.Object({ v: Type.String() }),
replay: "unsafe",
async execute(_a, _api, c) {
t.calls++;
await gate.wait(c);
return { content: [{ type: "text", text: "ran" }] };
},
};
let env = await open({ backend: "jsonl", tools: [t] });
onTestFinished(() => env.close());
await env.root.send({ content: "tool:x" }, ctx);
const task = await untilPhase(env, "pi.tool", "started");
await gate.arrivals(1);
await env.crash();
chop(join(env.dir!, "main.jsonl"), 1); // sidecar fsync'd, marker not
const s = await JsonlStorage.open(env.dir!, { fsync: false });
assert.equal((await s.task(task.id, ctx))?.checkpoint, undefined); // unconfirmed sidecar record ignored
await s.close();
env = await open({ dir: env.dir, backend: "jsonl", tools: [t] });
await untilPhase(env, "pi.tool", "started");
await gate.arrivals(2);
assert.equal(t.calls, 2); // once per process. (In a real crash the marker and the commit return are the same event, so a missing marker means execute never started; here the chop simulates that.)
gate.open();
await env.root.waitForIdle(ctx);
assert.match(JSON.stringify((await env.entries()).find((e) => e.kind === "pi.tool_result")!.model), /ran/);
});
test("an unpublished sidecar tail is truncated before its sequence is reused", async () => {
let env = await open({ backend: "jsonl" });
onTestFinished(() => env.close());
await env.root.config.set({ followUpMode: "all" } as never, ctx);
await env.crash();
assert.deepEqual(last(join(env.dir!, "main.jsonl")).refs, ["sticky-1.jsonl"]);
chop(join(env.dir!, "main.jsonl"), 1); // sticky N remains, but publication N is gone
env = await open({ dir: env.dir, backend: "jsonl" });
assert.equal((await env.root.sticky(ctx)).followUpMode, "one-at-a-time");
await env.root.config.set({ followUpMode: "all" } as never, ctx); // safely reuse N after replay truncated the old tail
await env.crash();
env = await open({ dir: env.dir, backend: "jsonl" }); // the duplicate-N poison used to fail here
assert.equal((await env.root.sticky(ctx)).followUpMode, "all");
const sequences = readFileSync(join(env.dir!, "sticky-1.jsonl"), "utf8")
.split("\n")
.filter(Boolean)
.map((line) => (JSON.parse(line) as { seq: number }).seq);
for (let index = 1; index < sequences.length; index++) assert.ok(sequences[index]! > sequences[index - 1]!);
});
test("a third-party kind's checkpoint + sticky write in one commit spans two sidecars → one marker; marker torn → both halves dropped together", async () => {
const gate = new Gate();
const ran: string[] = [];
let state!: Namespace<{ mark: string }>;
const k = defineTask<null, { phase: "a" } | { phase: "b" }, null, never, null>({
name: "two-files",
async initial() {
return { next: { phase: "a" } };
},
phases: {
async a(_t, rt, c) {
ran.push("a");
await rt.commit((tx) => {
tx.checkpoint({ phase: "b" });
tx.plugins(state).mark = "b";
}, c); // task sidecar + sticky sidecar, no main
await gate.wait(c);
return { done: () => ({ status: "completed", result: null }) };
},
async b(_t, _rt, c) {
ran.push("b");
await gate.wait(c);
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const setup = (harness: Parameters<NonNullable<Parameters<typeof open>[0]["setup"]>>[0]) => {
state = harness.namespace("test.atomicity", { sticky: { mark: "initial" } });
};
let env = await open({ backend: "jsonl", taskKinds: [k], setup });
onTestFinished(() => env.close());
const ref = await env.root.commit((tx) => tx.createTask(k, null, { conversationId: 1, background: true }), ctx);
await gate.arrivals(1);
await env.crash();
const main = last(join(env.dir!, "main.jsonl"));
assert.deepEqual(main.writes, []); // a marker with no main-table writes
assert.deepEqual([...main.refs!].sort(), [`sticky-1.jsonl`, `task-${ref.id}.jsonl`].sort());
assert.equal(main.seq, last(join(env.dir!, `task-${ref.id}.jsonl`)).seq);
assert.equal(main.seq, last(join(env.dir!, "sticky-1.jsonl")).seq);
// intact: replay sees phase b and the mark; the scheduler resumes INTO b
env = await open({ dir: env.dir, backend: "jsonl", taskKinds: [k], setup });
await gate.arrivals(2);
assert.deepEqual(ran, ["a", "b"]);
assert.equal((await env.root.sticky(ctx)).plugins["test.atomicity"]?.mark, "b");
await env.crash();
// torn: drop the horizon record → both sidecar halves are beyond the horizon and discarded together
assert.deepEqual(last(join(env.dir!, "main.jsonl")).writes, []); // a reopen that only reads appends nothing
chop(join(env.dir!, "main.jsonl"), 1);
// Open storage alone to see what replay restored, before any scheduler runs.
const s = await JsonlStorage.open(env.dir!, { fsync: false });
assert.equal((await s.task(ref.id, ctx))?.checkpoint?.phase, "a"); // the `b` record has no marker: dropped
assert.equal(
(
(await s.doc({ doc: "sticky", conversationId: 1 }, ctx)) as {
plugins: { "test.atomicity"?: { mark?: string } };
}
).plugins["test.atomicity"]?.mark,
undefined,
); // and so was its sticky half
await s.close();
// Resuming redoes phase a, which writes the pair again: the torn commit is repeated, not lost or half-applied.
env = await open({ dir: env.dir, backend: "jsonl", taskKinds: [k], setup });
await gate.arrivals(3);
assert.deepEqual(ran, ["a", "b", "a"]);
assert.equal((await env.root.sticky(ctx)).plugins["test.atomicity"]?.mark, "b");
gate.open();
});
test("torn tail: bytes after the last newline are truncated on open; a later append starts a fresh line", async () => {
let env = await open({ backend: "jsonl" });
onTestFinished(() => env.close());
await env.root.write({ kind: "note" }, ctx);
await env.crash();
appendFileSync(join(env.dir!, "main.jsonl"), '{"seq":99,"maxId":9,"wri');
env = await open({ dir: env.dir, backend: "jsonl" });
await env.root.write({ kind: "note" }, ctx);
await env.crash();
const lines = readFileSync(join(env.dir!, "main.jsonl"), "utf8").split("\n").filter(Boolean);
for (const l of lines) JSON.parse(l); // every line is a complete record
env = await open({ dir: env.dir, backend: "jsonl" });
assert.equal((await env.entries()).filter((e) => e.kind === "note").length, 2);
});
test("a malformed complete record fails open; a stale sidecar for a terminal task cannot resurrect state", async () => {
let env = await open({ backend: "jsonl" });
onTestFinished(() => env.close());
const ref = await env.root.commit(
(tx) => tx.createTask(kinds.plugin, { handler: "none", input: null }, { conversationId: 1, background: true }),
ctx,
);
await env.h.waitForTask(ref.id, ctx);
await env.crash();
// stale sidecar: a `running` record for the now-terminal task
writeFileSync(
join(env.dir!, `task-${ref.id}.jsonl`),
`${JSON.stringify({ seq: 1, maxId: 1, writes: [{ type: "task.patch", patch: { id: ref.id, status: "running", checkpoint: { phase: "started" } } }] })}\n`,
);
env = await open({ dir: env.dir, backend: "jsonl" });
assert.equal((await env.h.getTask(ref.id, ctx))?.status, "terminal");
await env.crash();
appendFileSync(join(env.dir!, "main.jsonl"), `${JSON.stringify({ seq: 1, maxId: 1, writes: [] })}\n`); // non-monotonic seq, complete line
await assert.rejects(JsonlStorage.open(env.dir!, { fsync: false }), /sequence not increasing/);
const good = readFileSync(join(env.dir!, "main.jsonl"), "utf8").split("\n").filter(Boolean);
writeFileSync(
join(env.dir!, "main.jsonl"),
`${good
.slice(0, -1)
.map((l) => `${l}\n`)
.join("")}{"seq":"x"}\n`,
);
await assert.rejects(JsonlStorage.open(env.dir!, { fsync: false }), /lacks seq/);
writeFileSync(
join(env.dir!, "main.jsonl"),
good
.slice(0, -1)
.map((l) => `${l}\n`)
.join(""),
);
});

View file

@ -0,0 +1,519 @@
import assert from "node:assert/strict";
import { Type } from "typebox";
import { onTestFinished, test } from "vitest";
import { Harness, kinds } from "../../../src/harness/pico3/harness.ts";
import { MemoryStorage } from "../../../src/harness/pico3/memory.ts";
import { defineSystemSection } from "../../../src/harness/pico3/system.ts";
import {
type CoreTx,
defineEntry,
defineTask,
Forbidden,
type Id,
type Namespace,
type RequestMessage,
type Runtime,
type ToolDeclaration,
} from "../../../src/harness/pico3/types.ts";
import { ctx, echoScript, fake, Gate, open, resultOf, tool } from "./helpers.ts";
// ---------------------------------------------------------------------------
// §1 registry and token identity
// ---------------------------------------------------------------------------
test("registry: a plugin kind named pi.* is rejected by defineTask and by open; a duplicate name is rejected; a forged core flag confers nothing", async () => {
assert.throws(
() =>
defineTask({
name: "pi.generation",
phases: {},
async initial() {
return { done: () => ({ status: "completed", result: null }) };
},
async abort() {
return () => null;
},
} as never),
/reserved/,
);
// bypass defineTask entirely
const forged = { ...kinds.job, name: "pi.generation", core: true };
await assert.rejects(
Harness.open(new MemoryStorage(), { models: fake({ respond: echoScript }), taskKinds: [forged as never] }, ctx),
/reserved/,
);
const dup = defineTask<null, { phase: "x" }, null, null, null>({
name: "dup",
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async initial() {
return { next: { phase: "x" } };
},
async abort() {
return () => null;
},
});
await assert.rejects(
Harness.open(new MemoryStorage(), { models: fake({ respond: echoScript }), taskKinds: [dup, { ...dup }] }, ctx),
/registered twice/,
);
// `core: true` on an ordinary kind: still cannot append entries
let forbidden: unknown;
const sneaky = {
...defineTask<null, { phase: "x" }, null, null, null>({
name: "sneaky",
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async initial(_t, rt, c) {
try {
await rt.commit((tx) => (tx as CoreTx).appendEntry(1, { kind: "x" }), c);
} catch (e) {
forbidden = e;
}
return { next: { phase: "x" } };
},
async abort() {
return () => null;
},
}),
core: true,
};
const env = await open({ taskKinds: [sneaky] });
onTestFinished(() => env.close());
const ref = await env.root.commit((tx) => tx.createTask(sneaky, null, { conversationId: 1, background: true }), ctx);
await env.h.waitForTask(ref.id, ctx);
assert.ok(forbidden instanceof Forbidden);
});
test("registry: stale/redeclared kind tokens reject for createTask and hooks; unsubscribe is idempotent and never removes a newer registration", async () => {
const k = defineTask<null, { phase: "x" }, null, null, null>({
name: "k",
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async initial() {
return { next: { phase: "x" } };
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [k], tools: [tool("x")] });
onTestFinished(() => env.close());
const hookNamespace = env.h.namespace("test.authority-hooks", {});
const stale = { ...k }; // same name, different object
await assert.rejects(
env.root.commit((tx) => tx.createTask(stale, null, { conversationId: 1 }), ctx),
/not the registered token/,
);
assert.throws(() => env.h.hooks(hookNamespace, stale, {}), /not the registered token/);
assert.throws(() => env.root.hooks(hookNamespace, { ...kinds.tool }, {}), /not the registered token/);
const seen: string[] = [];
const off1 = env.h.hooks(hookNamespace, kinds.tool, {
beforeTool: (c) => {
seen.push(`1:${c.name}`);
},
});
off1();
const off2 = env.h.hooks(hookNamespace, kinds.tool, {
beforeTool: (c) => {
seen.push(`2:${c.name}`);
},
});
off1(); // second call: must not splice off2
await (await env.root.send({ content: "tool:x" }, ctx)).wait(ctx);
assert.deepEqual(seen, ["2:x"]);
off2();
off2();
});
test("registry: duplicate tool/section/entry names reject; unregister removes only the exact object registered", async () => {
const env = await open({ tools: [tool("x")] });
onTestFinished(() => env.close());
assert.throws(() => env.h.registerTool(tool("x")), /already registered/);
const y1 = tool("y");
const off = env.h.registerTool(y1);
off();
const y2 = tool("y");
env.h.registerTool(y2);
off(); // idempotent: y2 stays
assert.throws(() => env.h.registerTool(tool("y")), /already registered/);
const s = defineSystemSection<string>({ key: "s", render: (v) => v });
const offS = env.h.registerSection(s);
assert.throws(
() => env.h.registerSection(defineSystemSection<string>({ key: "s", render: (v) => v })),
/already registered/,
);
offS();
offS();
const e = defineEntry("e");
env.h.registerEntryKind(e);
assert.throws(() => env.h.registerEntryKind(defineEntry("e")), /already registered/);
assert.throws(() => env.h.registerEntryKind(defineEntry("pi.mine")), /reserved/);
});
test("one owning Session per Storage object in-process", async () => {
const storage = new MemoryStorage();
const h = await Harness.open(storage, { models: fake({ respond: echoScript }) }, ctx);
onTestFinished(() => h.close(ctx));
await assert.rejects(
Harness.open(storage, { models: fake({ respond: echoScript }) }, ctx),
/already has an owning Session/,
);
await h.close(ctx);
const h2 = await Harness.open(storage, { models: fake({ respond: echoScript }) }, ctx); // released on close
await h2.close(ctx);
});
// ---------------------------------------------------------------------------
// §2 capability enforcement with malicious casts from an ordinary task
// ---------------------------------------------------------------------------
/** Runs `attempt` inside an ordinary task's commit with the tx cast to CoreTx; records what it threw. */
function probeKind(
name: string,
attempts: { [label: string]: (tx: CoreTx, rt: Runtime, cur: { conversationId: Id; id: Id }) => unknown },
results: Map<string, unknown>,
) {
return defineTask<{ target?: Id }, { phase: "x" }, null, null, null>({
name,
async initial(_t, rt, c) {
for (const [label, fn] of Object.entries(attempts)) {
try {
await rt.commit(async (tx, cur) => {
await fn(tx as unknown as CoreTx, rt, cur);
}, c);
results.set(label, "allowed");
} catch (e) {
results.set(label, e);
}
}
return { next: { phase: "x" } };
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
}
test("malicious casts: every core-only operation rejects at runtime for an ordinary task", async () => {
const results = new Map<string, unknown>();
const gate = new Gate();
let plugin!: Namespace<{ ok: boolean }>;
let stalePlugin!: Namespace<{ ok: boolean }>;
const other = defineTask<null, { phase: "x" }, null, null, null>({
name: "other",
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async initial(_t, _rt, c) {
await gate.wait(c);
return { next: { phase: "x" } };
},
async abort() {
return () => null;
},
});
const probe = probeKind(
"probe",
{
send: (tx) => tx.send(1, { content: "x" }),
boundary: (tx) => tx.boundary(1, "final", undefined),
resolveInputs: (tx) => tx.resolveInputs([1], { status: "done", answer: 1 }),
rewindable: (tx) => tx.rewindable(1),
sticky: (tx) => tx.sticky(1),
session: (tx) => tx.session(),
toolSlot: (tx) => tx.toolSlot({ conversationId: 1, input: { index: 0 } }),
appendEntry: (tx) => tx.appendEntry(1, { kind: "x" }),
createCoreByName: (tx) => tx.createTask({ kind: "pi.generation", conversationId: 1, input: { inputs: [] } }),
createCoreByToken: (tx) => tx.createTask(kinds.generation, { inputs: [] }, { conversationId: 1 }),
markTask: (tx) => tx.markTask(1),
writeHead: (tx) => tx.write(1, { kind: "x", head: "self" }),
writeReservedKind: (tx) => tx.write(1, { kind: "pi.assistant", model: [] }),
otherSlot: (tx) => {
const t = tx as unknown as { liveTasks: Map<Id, { id: Id; kind: string }> };
void t;
return tx.slot({ id: otherId, kind: other });
},
outOfSubtreeWrite: (tx) => tx.write(foreignConv, { kind: "x" }),
outOfSubtreeRead: (tx) => tx.newestEntry(foreignConv),
outOfSubtreeParent: (tx) => tx.createConversation({ parent: { conversationId: foreignConv, at: "start" } }),
outOfSubtreeConfig: (tx) => tx.config(foreignConv).get("profile"),
coreConfigWrite: (tx) =>
(tx.config(1) as unknown as { set(key: string, value: string): void }).set("profile", "forged"),
stalePlugin: (tx) => tx.plugins(stalePlugin),
ownWriteAllowed: (tx) => tx.write(1, { kind: "x" }),
ownPluginAllowed: (tx) => {
tx.plugins(plugin).ok = true;
},
},
results,
);
let otherId = 0;
let foreignConv = 0;
const env = await open({ taskKinds: [probe, other] });
onTestFinished(() => env.close());
stalePlugin = env.h.namespace("test.task-plugin", { sticky: { ok: false } });
stalePlugin.unregister();
plugin = env.h.namespace("test.task-plugin", { sticky: { ok: false } });
foreignConv = (await env.h.createConversation({}, ctx)).id;
otherId = (await env.root.commit((tx) => tx.createTask(other, null, { conversationId: 1, background: true }), ctx))
.id;
await gate.arrivals(1);
const ref = await env.root.commit((tx) => tx.createTask(probe, {}, { conversationId: 1, background: true }), ctx);
await env.h.waitForTask(ref.id, ctx);
gate.open();
const forbidden = [...results.entries()]
.filter(([, v]) => v instanceof Forbidden)
.map(([k]) => k)
.sort();
const allowed = [...results.entries()]
.filter(([, v]) => v === "allowed")
.map(([k]) => k)
.sort();
assert.deepEqual(allowed, ["ownPluginAllowed", "ownWriteAllowed"]);
assert.deepEqual(forbidden, [
"appendEntry",
"boundary",
"coreConfigWrite",
"createCoreByName",
"createCoreByToken",
"markTask",
"otherSlot",
"outOfSubtreeConfig",
"outOfSubtreeParent",
"outOfSubtreeRead",
"outOfSubtreeWrite",
"resolveInputs",
"rewindable",
"send",
"session",
"stalePlugin",
"sticky",
"toolSlot",
"writeHead",
"writeReservedKind",
]);
assert.equal(
results.size,
forbidden.length + allowed.length,
`unexpected outcomes: ${[...results.entries()]
.filter(([, v]) => v !== "allowed" && !(v instanceof Forbidden))
.map(([k, v]) => `${k}=${String(v)}`)
.join(", ")}`,
);
});
test("host tx: core operations reject via cast; host write/plugin state/config work", async () => {
const env = await open({});
onTestFinished(() => env.close());
const state = env.h.namespace<{ s: number }>("test.host-session", { session: { s: 0 } });
let escaped: CoreTx | undefined;
let escapedConfig: ReturnType<CoreTx["config"]> | undefined;
let callbackArguments = 0;
await env.root.commit((tx, ...args) => {
callbackArguments = 1 + args.length;
escaped = tx as unknown as CoreTx;
escapedConfig = tx.config(1);
const cast = tx as unknown as Record<string, unknown>;
for (const hidden of ["raw", "putInput", "appendEntryInternal", "changes", "invoker", "closing"])
assert.equal(cast[hidden], undefined, `${hidden} must not exist on the callback surface`);
assert.equal(Object.getPrototypeOf(tx), null);
}, ctx);
assert.equal(callbackArguments, 2, "internal transaction control must not reach a host callback");
assert.throws(() => escaped!.config(1), /outside its callback/);
assert.throws(() => escapedConfig!.get("profile"), /outside its callback/);
const core = (fn: (tx: CoreTx) => unknown) => env.root.commit((tx) => fn(tx as unknown as CoreTx), ctx);
await assert.rejects(
core((tx) => tx.send(1, { content: "x" })),
Forbidden,
);
await assert.rejects(
core((tx) => tx.appendEntry(1, { kind: "x" })),
Forbidden,
);
await assert.rejects(
core((tx) => tx.sticky(1)),
Forbidden,
);
await assert.rejects(
core((tx) => tx.boundary(1, "final", undefined)),
Forbidden,
);
await assert.rejects(
core((tx) => tx.createTask(kinds.generation, { inputs: [] }, { conversationId: 1 })),
Forbidden,
);
await assert.rejects(
core((tx) => tx.checkpoint({ phase: "x" })),
Forbidden,
);
await assert.rejects(
core((tx) => tx.slot({ id: 1, kind: kinds.job })),
Forbidden,
);
await env.root.commit((tx) => {
tx.plugins(state).s = 1;
tx.config(1).set("profile", "p2");
}, ctx);
assert.equal((await env.root.config.get(ctx)).profile, "p2");
});
// ---------------------------------------------------------------------------
// invocation tokens
// ---------------------------------------------------------------------------
test("an ordinary runtime cannot abort a task in an unrelated conversation", async () => {
let target = 0;
let result: unknown;
const attacker = defineTask<null, { phase: "never" }, null, null, null>({
name: "abort-attacker",
async initial(_task, runtime, context) {
try {
result = await runtime.abortTask(target, context);
} catch (error) {
result = error;
}
return { done: () => ({ status: "completed", result: null }) };
},
phases: {
async never() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const gate = new Gate();
const env = await open({ taskKinds: [attacker], models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const active = await env.root.send({ content: "root" }, ctx);
await gate.arrivals(1);
target = (await env.tasks()).find((task) => task.kind === "pi.generation" && task.status !== "terminal")!.id;
const foreign = await env.h.createConversation({}, ctx);
const ref = await foreign.commit(
(tx) => tx.createTask(attacker, null, { conversationId: foreign.id, background: true }),
ctx,
);
await env.h.waitForTask(ref.id, ctx);
assert.ok(result instanceof Forbidden);
assert.equal((await env.h.getTask(target, ctx))?.abort, undefined);
gate.open();
assert.equal((await active.wait(ctx)).status, "done");
});
test("a captured runtime cannot commit after its invocation returned, after terminalization, or (run mode) after a durable mark", async () => {
let captured: Runtime | undefined;
const gate = new Gate();
const k = defineTask<null, { phase: "x" }, null, null, null>({
name: "cap",
async initial(_t, rt, c) {
captured = rt;
await gate.wait(c);
return { next: { phase: "x" } };
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [k] });
onTestFinished(() => env.close());
const ref = await env.root.commit((tx) => tx.createTask(k, null, { conversationId: 1, background: true }), ctx);
await gate.arrivals(1);
// marked while running: the run invocation's commits reject
await env.h.markTask(ref.id, ctx);
await assert.rejects(
captured!.commit(() => 1, ctx),
/marked run invocation|finished invocation/,
); // the mark also signals the invocation, which may already have wound down
gate.open();
await env.h.waitForTask(ref.id, ctx);
await assert.rejects(
captured!.commit(() => 1, ctx),
/finished invocation|not live/,
);
// a second kind whose invocation returns normally
let cap2: Runtime | undefined;
let child = 0;
const k2 = defineTask<null, { phase: "x" }, null, null, null>({
name: "cap2",
async initial(_t, rt, context) {
cap2 = rt;
child = await rt.createOwnedConversation({}, context);
return { next: { phase: "x" } };
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env2 = await open({ taskKinds: [k2] });
onTestFinished(() => env2.close());
const r2 = await env2.root.commit((tx) => tx.createTask(k2, null, { conversationId: 1, background: true }), ctx);
await env2.h.waitForTask(r2.id, ctx);
await assert.rejects(
cap2!.commit(() => 1, ctx),
Forbidden,
);
await assert.rejects(cap2!.createOwnedConversation({}, ctx), Forbidden);
await assert.rejects(cap2!.sendOwned(child, { content: "late" }, ctx), Forbidden);
});
test("ToolDeclaration receives typed Static args; TypeBox 1.x validation rejects bad calls with a synthetic result", async () => {
const schema = Type.Object({ n: Type.Number(), tags: Type.Optional(Type.Array(Type.String())) });
const t: ToolDeclaration<typeof schema> = {
name: "typed",
description: "",
parameters: schema,
async execute(args) {
return { content: [{ type: "text", text: `n=${args.n * 2} tags=${args.tags?.length ?? 0}` }] };
},
};
const respond = (m: RequestMessage[]) => {
const last = [...m].reverse().find((x) => x.role === "user" || x.role === "toolResult")!;
if (last.role === "user") {
const c = String((last as { content: unknown }).content);
if (c === "good") return { toolCalls: [{ name: "typed", arguments: { n: 21, tags: ["a"] } }] };
if (c === "bad") return { toolCalls: [{ name: "typed", arguments: { n: "x" } }] };
}
return { text: "ok" };
};
const env = await open({ tools: [t], models: fake({ respond }) });
onTestFinished(() => env.close());
await (await env.root.send({ content: "good" }, ctx)).wait(ctx);
assert.match(
JSON.stringify((await env.entries()).filter((e) => e.kind === "pi.tool_result").pop()!.model),
/n=42 tags=1/,
);
await (await env.root.send({ content: "bad" }, ctx)).wait(ctx);
const bad = (await env.entries()).filter((e) => e.kind === "pi.tool_result").pop()!;
assert.match(JSON.stringify(bad.model), /invalid arguments: \/n: must be number/);
assert.equal((bad.data as { diagnostics: { code: string }[] }).diagnostics[0]!.code, "invalid_arguments");
const tasks = (await env.tasks()).filter((x) => x.kind === "pi.tool");
assert.equal(resultOf<{ entry: number }>(tasks[1])?.entry, bad.id);
});

View file

@ -0,0 +1,307 @@
import assert from "node:assert/strict";
import { Type } from "typebox";
import { onTestFinished, test } from "vitest";
import { kinds } from "../../../src/harness/pico3/harness.ts";
import { defineTask, type Runtime, TaskContractFault } from "../../../src/harness/pico3/types.ts";
import { ctx, echoScript, fake, fakeHost, Gate, model, open, resultOf } from "./helpers.ts";
// ---------------------------------------------------------------------------
// §3 prospective busy()
// ---------------------------------------------------------------------------
test("overflow with nothing collapsible: the notice is appended by the terminal closure (the closing generation counts as gone); no inbox item is stranded", async () => {
const models = fake({ respond: echoScript });
const modelObj = models.resolve({ provider: "", modelId: "" })! as { contextWindow: number; maxTokens: number };
const env = await open({ models, root: { rewindable: { model, keepRecent: 1000 } } });
onTestFinished(() => env.close());
modelObj.contextWindow = 50;
modelObj.maxTokens = 10;
const a = await env.root.send({ content: "x".repeat(2000) }, ctx);
const r = await a.wait(ctx);
assert.equal(r.reason, "failed");
assert.equal(r.detail, "overflow");
const es = await env.entries();
assert.ok(
es.some((e) => e.kind === "pi.notice"),
"notice appended",
);
assert.deepEqual((await env.root.sticky(ctx)).inbox, []);
assert.deepEqual((await env.root.sticky(ctx)).turn, { tools: [] });
});
test("a job's terminal-closure notification on an idle conversation appends immediately (a background job is not busy)", async () => {
const host = fakeHost();
const env = await open({ processHost: host });
onTestFinished(() => env.close());
const ref = await env.root.commit(
(tx) =>
tx.createTask(
kinds.job,
{ command: "x", args: [], cwd: "/", notify: true, rerun: false },
{ conversationId: 1, background: true },
),
ctx,
);
await new Promise((r) => setTimeout(r, 30));
host.exit(`${ref.id}:1`, 0);
const t = await env.h.waitForTask(ref.id, ctx);
assert.equal(t.outcome?.status, "completed");
assert.ok((await env.entries()).some((e) => e.kind === "pi.notice"));
assert.deepEqual((await env.root.sticky(ctx)).inbox, []);
});
test("a same-batch successor generation keeps the conversation busy: a job notification landing in that commit window is queued, then placed at the next boundary", async () => {
// post_tools creates the continuation in its own terminal batch; a write during that batch sees busy.
const gate = new Gate();
const host = fakeHost();
let inbox: unknown;
const k = defineTask<null, { phase: "x" }, null, null, null>({
name: "peek",
async initial(_t, rt, c) {
inbox = await rt.commit(
(tx, cur) => [
tx.snapshot({ doc: "sticky", conversationId: cur.conversationId }).inbox.length,
tx.snapshot({ doc: "sticky", conversationId: cur.conversationId }).turn.tools.length,
],
c,
);
return { next: { phase: "x" } };
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({
tools: [
{
name: "slow",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute(_a, _api, c) {
await gate.wait(c);
return { content: [{ type: "text", text: "ok" }] };
},
},
],
processHost: host,
taskKinds: [k],
});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "tool:slow" }, ctx);
await gate.arrivals(1);
// while the tool runs (busy), a job finishes and wants to notify
const job = await env.root.commit(
(tx) =>
tx.createTask(
kinds.job,
{ command: "x", args: [], cwd: "/", notify: true, rerun: false },
{ conversationId: 1, background: true },
),
ctx,
);
await new Promise((r) => setTimeout(r, 30));
host.exit(`${job.id}:1`, 0);
await env.h.waitForTask(job.id, ctx);
assert.equal((await env.root.sticky(ctx)).inbox.length, 1); // queued: the turn is busy
gate.open();
await a.wait(ctx);
assert.deepEqual((await env.root.sticky(ctx)).inbox, []); // placed at the post-tools boundary
const ks = (await env.entries()).map((e) => e.kind.replace("pi.", ""));
assert.ok(
ks.indexOf("notice") > ks.indexOf("tool_result") && ks.indexOf("notice") < ks.lastIndexOf("assistant"),
ks.join(" "),
);
void inbox;
});
test("nothing is stranded when the last turn task terminalizes: a write that arrives during the final closure lands in the transcript", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
await gate.arrivals(1);
const w = await env.root.write({ kind: "note" }, ctx); // queued
gate.open();
await a.wait(ctx);
assert.equal((await env.input(w))?.status, "done");
assert.deepEqual((await env.root.sticky(ctx)).inbox, []);
});
// ---------------------------------------------------------------------------
// §6 phase-map contract
// ---------------------------------------------------------------------------
test("phase roles: a `next` transition into an in-flight phase is a contract fault → outcome faulted; the kind's declared failure type is never fabricated", async () => {
const reports: unknown[] = [];
const k = defineTask<null, { phase: "fly" } | { phase: "ok" }, null, { reason: "declared" }, null>({
name: "roles",
inflight: ["fly"],
async initial() {
return { next: { phase: "fly" } };
},
phases: {
async fly() {
return { done: () => ({ status: "failed", failure: { reason: "declared" } }) };
},
async ok() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [k], onReport: (e) => reports.push(e) });
onTestFinished(() => env.close());
const ref = await env.root.commit((tx) => tx.createTask(k, null, { conversationId: 1, background: true }), ctx);
const t = await env.h.waitForTask(ref.id, ctx);
assert.equal(t.outcome?.status, "faulted");
assert.match((t.outcome as { error: string }).error, /in-flight phase fly/);
assert.ok(reports[0] instanceof TaskContractFault);
});
test("an in-flight phase written with rt.commit before the effect is entered after reopen only", async () => {
const entered: string[] = [];
const gate = new Gate();
const k = defineTask<null, { phase: "fly" }, null, null, null>({
name: "fly",
inflight: ["fly"],
async initial(_t, rt, c) {
await rt.commit((tx) => tx.checkpoint({ phase: "fly" }), c);
entered.push("initial");
await gate.wait(c);
return { done: () => ({ status: "completed", result: null }) };
},
phases: {
async fly() {
entered.push("fly");
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
let env = await open({ backend: "jsonl", taskKinds: [k] });
onTestFinished(() => env.close());
const ref = await env.root.commit((tx) => tx.createTask(k, null, { conversationId: 1, background: true }), ctx);
await gate.arrivals(1);
await env.crash();
env = await open({ dir: env.dir, backend: "jsonl", taskKinds: [k] });
await env.h.waitForTask(ref.id, ctx);
assert.deepEqual(entered, ["initial", "fly"]);
});
test("an ordinary kind that throws, or returns an invalid step/completion, ends faulted (not a declared failure); the task stays readable", async () => {
const reports: unknown[] = [];
const thrower = defineTask<null, { phase: "x" }, null, { reason: "declared" }, null>({
name: "thrower",
async initial() {
throw new Error("kaboom");
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const badStep = defineTask<null, { phase: "x" }, null, null, null>({
name: "badstep",
async initial() {
return { nope: true } as never;
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const badDone = defineTask<null, { phase: "x" }, null, null, null>({
name: "baddone",
async initial() {
return { done: () => ({ status: "weird" }) as never };
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const badPhase = defineTask<null, { phase: "x" }, null, null, null>({
name: "badphase",
async initial() {
return { next: { phase: "zzz" } as never };
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [thrower, badStep, badDone, badPhase], onReport: (e) => reports.push(e) });
onTestFinished(() => env.close());
for (const [k, re] of [
[thrower, /kaboom/],
[badStep, /invalid step/],
[badDone, /invalid completion/],
[badPhase, /unknown phase zzz/],
] as const) {
const ref = await env.root.commit(
(tx) => tx.createTask(k as typeof thrower, null, { conversationId: 1, background: true }),
ctx,
);
const t = await env.h.waitForTask(ref.id, ctx);
assert.equal(t.outcome?.status, "faulted", k.name);
assert.match((t.outcome as { error: string }).error, re);
assert.equal((await env.h.getTask(ref.id, ctx))?.status, "terminal");
}
assert.equal(reports.length, 4);
assert.equal(resultOf(await env.h.getTask(1, ctx)), undefined);
});
test("transition builder may return a checkpoint, a completion, or 'retry' atomically", async () => {
let tries = 0;
const k = defineTask<null, { phase: "x" }, { done: true }, null, null>({
name: "builder",
async initial() {
return {
next: (_tx) => {
tries++;
return tries < 3 ? "retry" : { phase: "x" };
},
};
},
phases: {
async x() {
return { next: () => ({ status: "completed", result: { done: true } }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [k] });
onTestFinished(() => env.close());
const ref = await env.root.commit((tx) => tx.createTask(k, null, { conversationId: 1, background: true }), ctx);
const t = await env.h.waitForTask(ref.id, ctx);
assert.equal(tries, 3);
assert.deepEqual(t.outcome, { status: "completed", result: { done: true } });
void (null as unknown as Runtime);
});

View file

@ -0,0 +1,112 @@
import assert from "node:assert/strict";
import { replicatedState } from "@earendil-works/chord";
import { onTestFinished, test } from "vitest";
import {
attachChordView,
createPicoConversationService,
type PublishedConversationView,
} from "../../../src/harness/pico3/chord.ts";
import type { Envelope, JsonObject } from "../../../src/harness/pico3/types.ts";
import { ctx, open, sleep } from "./helpers.ts";
test("conversation observation reports loaded and newly committed conversations off the Session line", async () => {
const env = await open({});
onTestFinished(() => env.close());
const ids: number[] = [];
const stop = env.h.onConversation((conversation) => ids.push(conversation.id));
onTestFinished(stop);
assert.deepEqual(ids, [env.root.id]);
const child = await env.h.createConversation({}, ctx);
assert.deepEqual(ids, [env.root.id, child.id]);
stop();
await env.h.createConversation({}, ctx);
assert.deepEqual(ids, [env.root.id, child.id]);
});
test("Chord bridge publishes exactly once per Pico envelope and converges to a fresh snapshot", async () => {
const env = await open({});
onTestFinished(() => env.close());
const raw = await env.root.watch(ctx);
const envelopes: Envelope[] = [];
raw.start((envelope) => envelopes.push(envelope));
const bridge = await attachChordView(env.root, replicatedState, ctx);
onTestFinished(() => bridge.close());
const service = createPicoConversationService(env.h, env.root, bridge.view);
const deliveries: { sequence: number; events: PublishedConversationView["commit"]["events"] }[] = [];
const unsubscribe = bridge.view.subscribe((value, _context, delivery) => {
if (delivery.kind === "update") deliveries.push({ sequence: delivery.sequence, events: value.commit.events });
});
onTestFinished(unsubscribe);
await (await env.root.send({ content: "hello" }, ctx)).wait(ctx);
await env.root.waitForIdle(ctx);
await sleep(0);
assert.equal(deliveries.length, envelopes.length);
assert.deepEqual(
deliveries.map((delivery) => delivery.sequence),
deliveries.map((_, index) => index + 1),
);
assert.deepEqual(
deliveries.map((delivery) => delivery.events),
envelopes.map((envelope) => envelope.events),
);
const invalidPatches: JsonObject[] = [
{ threshold: "not-a-number" },
{ selectedTools: "not-an-array" },
{ retry: 5 },
{ model: null },
{ steeringMode: "sometimes" },
];
for (const patch of invalidPatches) await assert.rejects(service.configSet(patch, ctx), /invalid config value/);
await assert.rejects(
service.fork("start", { rewindable: { threshold: "not-a-number" } } as never, ctx),
/invalid rewindable config value/,
);
await service.configSet({ threshold: 123, selectedTools: ["bash"], steeringMode: "all" }, ctx);
assert.equal((await env.root.config.get(ctx)).threshold, 123);
const foreign = await env.h.createConversation({}, ctx);
const foreignInput = await foreign.send({ content: "foreign" }, ctx);
await assert.rejects(service.inputAbort(foreignInput.id, ctx), /outside conversation/);
await foreignInput.wait(ctx);
const fresh = await env.root.watch(ctx);
fresh.stop();
const { commit: _commit, ...published } = bridge.view.value;
assert.deepEqual(published, fresh.view);
raw.stop();
});
test("Chord publication failure closes only the bridge and never rejects the persisted writer", async () => {
const env = await open({});
onTestFinished(() => env.close());
let reported: Error | undefined;
const bridge = await attachChordView(
env.root,
(initial) => {
const state = replicatedState(initial);
return {
get value() {
return state.value;
},
get state() {
return state.state;
},
subscribe: (listener) => state.subscribe(listener),
publish() {
throw new Error("publish failed");
},
};
},
ctx,
{
onFailure: (error) => {
reported = error;
},
},
);
onTestFinished(() => bridge.close());
const input = await env.root.send({ content: "persist anyway" }, ctx);
assert.equal((await input.wait(ctx)).status, "done");
await sleep(0);
assert.equal(bridge.closed, true);
assert.match(reported?.message ?? "", /publish failed/);
});

View file

@ -0,0 +1,205 @@
import assert from "node:assert/strict";
import { test } from "vitest";
import { Bounded } from "../../../src/harness/pico3/bounded.ts";
import { Harness, kinds } from "../../../src/harness/pico3/harness.ts";
import { Membrane } from "../../../src/harness/pico3/membrane.ts";
import { MemoryStorage } from "../../../src/harness/pico3/memory.ts";
import { defineTask, Forbidden, type JsonObject, type JsonValue } from "../../../src/harness/pico3/types.ts";
import { ctx, fake, Gate, model, open, sleep } from "./helpers.ts";
test("MemoryStorage reads cannot mutate authoritative records", async () => {
const storage = new MemoryStorage();
await storage.commit(
[
{ type: "conversation", conversation: { id: 1 } },
{ type: "entry", entry: { id: 2, conversationId: 1, kind: "original" } },
],
ctx,
);
const first = (await storage.entries([2], ctx)).get(2)!;
(first as { kind: string }).kind = "mutated";
assert.equal((await storage.entries([2], ctx)).get(2)!.kind, "original");
});
test("membrane rejects a wrapper hidden inside an assigned object and clones plain inputs", () => {
const target: { left: JsonObject; right: JsonObject } = { left: {}, right: { value: 1 } };
const membrane = new Membrane("nested");
const view = membrane.wrap(target);
const escaped = view.right;
assert.throws(() => {
view.left.payload = { escaped } as JsonValue;
}, /document proxy/);
const assigned = { nested: { value: 2 } };
view.left.payload = assigned;
assigned.nested.value = 3;
assert.deepEqual(target.left.payload, { nested: { value: 2 } });
membrane.revoke();
assert.throws(() => view.left, /outside its transaction/);
});
test("tail bounds account for partial multi-chunk and single-chunk trimming", () => {
const bytes = new Bounded(10, 100, "tail");
bytes.push(new TextEncoder().encode("abcdefgh"));
bytes.push(new TextEncoder().encode("ijklmnop"));
assert.equal(bytes.text(), "ghijklmnop");
assert.equal(bytes.droppedBytes, 6);
const lines = new Bounded(100, 2, "tail");
lines.push(new TextEncoder().encode("a\nb\nc\nd\n"));
assert.equal(lines.text(), "c\nd\n");
assert.equal(lines.droppedLines, 2);
});
test("terminal provider failure advances a queued follow-up without another send", async () => {
const gate = new Gate();
const env = await open({
models: fake({ respond: () => ({ error: "fatal" }), gate }),
root: { rewindable: { model }, sticky: { retry: { enabled: false, maxRetries: 0, baseDelayMs: 1 } } },
});
try {
const first = await env.root.send({ content: "first" }, ctx);
await gate.arrivals(1);
const queued = await env.root.send({ content: "second" }, ctx);
gate.open();
await first.wait(ctx);
await sleep(20);
const current = await queued.result(ctx);
assert.notEqual(current?.status, "queued");
assert.ok(
(await env.tasks()).some(
(task) => task.kind === "pi.generation" && (task.input as { inputs: number[] }).inputs.includes(queued.id),
),
);
} finally {
gate.open();
await env.close();
}
});
test("ordinary Runtime off-line reads enforce conversation subtree scope", async () => {
let observed: unknown;
const probe = defineTask<{ foreign: number }, { phase: "done" }, null, null, null>({
name: "scope-probe",
async initial(task, runtime, context) {
try {
observed = await runtime.context(task.input.foreign, undefined, context);
} catch (error) {
observed = error;
}
return { next: { phase: "done" } };
},
phases: {
async done() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [probe] });
try {
const foreign = await env.h.createConversation({}, ctx);
await foreign.write({ kind: "foreign" }, ctx);
const ref = await env.root.commit(
(tx) => tx.createTask(probe, { foreign: foreign.id }, { conversationId: 1, background: true }),
ctx,
);
await env.h.waitForTask(ref.id, ctx);
assert.ok(observed instanceof Forbidden, `ordinary task read foreign conversation: ${JSON.stringify(observed)}`);
} finally {
await env.close();
}
});
test("unknown live task kinds are orphaned when resume begins", async () => {
const storage = new MemoryStorage();
await storage.commit(
[
{ type: "conversation", conversation: { id: 1 } },
{ type: "doc", ref: { doc: "rewindable", conversationId: 1 }, ops: [["r", { plugins: {} }]] },
{
type: "doc",
ref: { doc: "sticky", conversationId: 1 },
ops: [["r", { inbox: [], turn: { tools: [] }, tasks: {}, plugins: {} }]],
},
{
type: "task",
task: {
id: 2,
conversationId: 1,
kind: "missing.kind",
input: null,
status: "running",
after: [],
owns: [],
},
},
],
ctx,
);
const harness = await Harness.open(storage, { models: fake({ respond: () => ({ text: "ok" }) }) }, ctx);
try {
harness.resume();
const task = await harness.waitForTask(2, ctx);
assert.equal(task.status, "terminal");
assert.equal(task.outcome?.status, "orphaned");
} finally {
await harness.close(ctx);
}
});
test("a task kind registered after open but before resume recovers its durable tasks", async () => {
const storage = new MemoryStorage();
await storage.commit(
[
{ type: "conversation", conversation: { id: 1 } },
{ type: "doc", ref: { doc: "rewindable", conversationId: 1 }, ops: [["r", { plugins: {} }]] },
{
type: "doc",
ref: { doc: "sticky", conversationId: 1 },
ops: [["r", { inbox: [], turn: { tools: [] }, tasks: {}, plugins: {} }]],
},
{
type: "task",
task: {
id: 2,
conversationId: 1,
kind: "recover.after-open",
input: null,
status: "running",
after: [],
owns: [],
},
},
],
ctx,
);
const recovered = defineTask<null, { phase: "finish" }, string, null, null>({
name: "recover.after-open",
async initial() {
return { next: { phase: "finish" } };
},
phases: {
async finish() {
return { done: () => ({ status: "completed", result: "replacement" }) };
},
},
async abort() {
return () => null;
},
});
const harness = await Harness.open(storage, { models: fake({ respond: () => ({ text: "ok" }) }) }, ctx);
try {
harness.registerTaskKind(recovered);
harness.resume();
assert.deepEqual((await harness.waitForTask(2, ctx)).outcome, {
status: "completed",
result: "replacement",
});
} finally {
await harness.close(ctx);
}
});
void kinds;

View file

@ -0,0 +1,409 @@
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Context } from "@earendil-works/chord";
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import type { AssistantMessage, AssistantMessageEvent, ToolCall } from "@earendil-works/pi-ai";
import { Type } from "typebox";
import {
kinds as builtin,
type ConversationHandle,
type Envelope,
Harness,
type HarnessOptions,
} from "../../../src/harness/pico3/harness.ts";
import { JsonlStorage } from "../../../src/harness/pico3/jsonl.ts";
import type { CollapseHooks } from "../../../src/harness/pico3/kinds/collapse.ts";
import type { GenerationHooks } from "../../../src/harness/pico3/kinds/generation.ts";
import type { PostToolsHooks } from "../../../src/harness/pico3/kinds/post-tools.ts";
import type { ToolHooks } from "../../../src/harness/pico3/kinds/tool.ts";
import { MemoryStorage } from "../../../src/harness/pico3/memory.ts";
import type { SystemSection } from "../../../src/harness/pico3/system.ts";
import type {
AnyKind,
CoreTx,
Entry,
Input,
JsonValue,
Models,
ProcessHost,
ProcessStatus,
RequestMessage,
Storage,
Task,
ToolDeclaration,
ToolResult,
} from "../../../src/harness/pico3/types.ts";
export const failureOf = (t: Task | undefined) =>
(t?.outcome as unknown as { failure: { reason: string; detail?: string } } | undefined)?.failure;
export const resultOf = <T>(t: Task | undefined) => (t?.outcome as unknown as { result: T } | undefined)?.result;
export type { CoreTx };
/** Test convenience: one object keyed by kind, registered harness-wide. */
export interface HooksByKind {
generation?: Partial<GenerationHooks>;
tool?: Partial<ToolHooks>;
postTools?: Partial<PostToolsHooks>;
collapse?: Partial<CollapseHooks>;
}
export const ctx = BACKGROUND_CONTEXT;
export const model = { provider: "anthropic", modelId: "fake-1" };
export const sleep = (ms: number) => new Promise((r) => setTimeout(r, ms));
export const kinds = (entries: Entry[]) =>
entries.map((e) => e.kind.replace("pi.", "") + (e.head ? "*" : "")).join(" ");
export const contentOf = (e: Entry | undefined): string => {
const c = (e?.model?.[0] as { content?: unknown } | undefined)?.content;
return typeof c === "string" ? c : JSON.stringify(c ?? null);
};
// ---------------------------------------------------------------------------
// Gate: a promise the test controls. `await g.wait(ctx)` blocks until open().
// ---------------------------------------------------------------------------
export class Gate {
private opened = false;
private waiters: (() => void)[] = [];
arrived = 0;
open() {
this.opened = true;
for (const w of this.waiters.splice(0)) w();
}
close() {
this.opened = false;
}
wait(ctx: Context): Promise<void> {
this.arrived++;
if (this.opened) return Promise.resolve();
return new Promise((resolve, reject) => {
this.waiters.push(resolve);
ctx.abortSignal?.addEventListener("abort", () => reject(ctx.abortSignal?.reason ?? new Error("aborted")), {
once: true,
});
});
}
/** Resolve once `n` callers have arrived. */
async arrivals(n: number, timeoutMs = 2000) {
const t0 = Date.now();
while (this.arrived < n) {
if (Date.now() - t0 > timeoutMs) throw new Error(`gate: only ${this.arrived}/${n} arrivals`);
await sleep(2);
}
}
}
// ---------------------------------------------------------------------------
// Fake provider. `respond` returns what to stream; `gate` pauses before the
// first token so a test can crash the harness in `requesting`.
// ---------------------------------------------------------------------------
export type Response = {
text?: string;
toolCalls?: { name: string; arguments: unknown }[];
error?: string;
stop?: "stop" | "length";
};
export interface FakeOptions {
respond(messages: RequestMessage[], call: number): Response;
gate?: Gate;
/** Only gate requests matching this predicate (default: all). */
gateWhen?: (messages: RequestMessage[]) => boolean;
tokenDelayMs?: number;
}
const usage = (i: number, o: number): AssistantMessage["usage"] => ({
input: i,
output: o,
cacheRead: 0,
cacheWrite: 0,
totalTokens: i + o,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
});
export function fake(o: FakeOptions): Models & { calls: number; requests: RequestMessage[][] } {
const m = {
id: "fake-1",
name: "Fake",
api: "anthropic-messages" as const,
provider: "anthropic" as const,
baseUrl: "",
reasoning: false,
input: ["text" as const],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 200_000,
maxTokens: 8192,
};
const api = {
calls: 0,
requests: [] as RequestMessage[][],
resolve: () => m,
async *stream(
_: unknown,
request: { messages: RequestMessage[] },
ctx: Context,
): AsyncIterable<AssistantMessageEvent> {
const n = api.calls++;
api.requests.push(request.messages);
const r = o.respond(request.messages, n);
const partial: AssistantMessage = {
role: "assistant",
content: [],
api: m.api,
provider: m.provider,
model: m.id,
usage: usage(0, 0),
stopReason: "stop",
timestamp: Date.now(),
};
yield { type: "start", partial };
if (o.gate && (o.gateWhen?.(request.messages) ?? true)) await o.gate.wait(ctx);
if (r.error !== undefined) {
yield { type: "error", reason: "error", error: { ...partial, stopReason: "error", errorMessage: r.error } };
return;
}
let index = 0;
let tokens = 0;
if (r.text !== undefined) {
partial.content.push({ type: "text", text: "" });
yield { type: "text_start", contentIndex: index, partial };
for (const word of r.text.split(" ")) {
if (o.tokenDelayMs) await sleep(o.tokenDelayMs);
ctx.abortSignal?.throwIfAborted();
const delta = `${word} `;
(partial.content[index] as { text: string }).text += delta;
tokens++;
yield { type: "text_delta", contentIndex: index, delta, partial };
}
yield {
type: "text_end",
contentIndex: index,
content: (partial.content[index] as { text: string }).text,
partial,
};
index++;
}
for (const tc of r.toolCalls ?? []) {
const call: ToolCall = {
type: "toolCall",
id: `call_${n}_${index}`,
name: tc.name,
arguments: tc.arguments as Record<string, unknown>,
};
partial.content.push(call);
yield { type: "toolcall_start", contentIndex: index, partial };
yield { type: "toolcall_end", contentIndex: index, toolCall: call, partial };
index++;
}
const stopReason = r.toolCalls?.length ? "toolUse" : (r.stop ?? "stop");
yield {
type: "done",
reason: stopReason,
message: { ...partial, usage: usage(request.messages.length * 50, tokens), stopReason },
};
},
async fetchDeferred(): Promise<AssistantMessage> {
throw new Error("no deferred");
},
async cancelDeferred() {},
};
return api;
}
/** Common script: tool calls when the user says "tool:<name>", otherwise an answer echoing the user. */
/** The last non-system message: system messages sit inside `messages` at their historical positions. */
export const lastMessage = (messages: RequestMessage[]) => [...messages].reverse().find((m) => m.role !== "system")!;
export const echoScript = (messages: RequestMessage[]): Response => {
const last = lastMessage(messages);
if (last.role === "toolResult") return { text: "after tools" };
const user = String((last as { content?: unknown }).content ?? "");
if (user.startsWith("tool:"))
return {
toolCalls: user
.slice(5)
.split(",")
.map((name) => ({ name, arguments: { v: name } })),
};
if (user.startsWith("error:")) return { error: user.slice(6) };
return { text: `answer to ${user}` };
};
// ---------------------------------------------------------------------------
// Tools
// ---------------------------------------------------------------------------
const toolSchema = Type.Object({ v: Type.String() });
export function tool(
name: string,
opts: {
replay?: "safe" | "unsafe";
gate?: Gate;
result?: Partial<ToolResult>;
throws?: string;
output?: ToolDeclaration["output"];
} = {},
): ToolDeclaration<typeof toolSchema> & { calls: number } {
const t = {
calls: 0,
name,
description: name,
parameters: toolSchema,
replay: opts.replay ?? "safe",
...(opts.output ? { output: opts.output } : {}),
async execute(args: { v: string }, _api: unknown, ctx: Context): Promise<ToolResult> {
t.calls++;
if (opts.gate) await opts.gate.wait(ctx);
if (opts.throws) throw new Error(opts.throws);
return { content: [{ type: "text", text: `${name}(${args.v})` }], ...opts.result };
},
};
return t;
}
// ---------------------------------------------------------------------------
// Fake process host
// ---------------------------------------------------------------------------
export function fakeHost() {
const procs = new Map<string, { status: ProcessStatus; started: number }>();
const host: ProcessHost & {
procs: typeof procs;
forget(key: string): void;
exit(key: string, code: number): void;
startCalls: number;
} = {
procs,
startCalls: 0,
async start(key) {
host.startCalls++;
if (!procs.has(key))
procs.set(key, {
status: { status: "running", stdout: "", stderr: "", droppedStdout: 0, droppedStderr: 0 },
started: Date.now(),
});
},
async status(key) {
return procs.get(key)?.status ?? { status: "unknown" };
},
async kill() {},
forget: (key) => procs.delete(key),
exit: (key, exitCode) => {
const p = procs.get(key);
if (p)
p.status = {
status: "exited",
exitCode,
stdout: `out ${key}`,
stderr: "",
droppedStdout: 0,
droppedStderr: 0,
};
},
};
return host;
}
// ---------------------------------------------------------------------------
// Harness factory with crash/reopen for JSONL.
// ---------------------------------------------------------------------------
export interface Env {
h: Harness<AnyKind[]>;
root: ConversationHandle<Record<string, JsonValue | undefined>>;
storage: Storage;
dir?: string;
entries(conversationId?: number): Promise<Entry[]>; // ascending
tasks(conversationId?: number): Promise<Task[]>;
input(id: number): Promise<Input | undefined>;
/** Simulate a crash: signal invocations, close storage, write nothing. Returns a reopener. */
crash(): Promise<() => Promise<Env>>;
close(): Promise<void>;
}
export async function open(
opts: Partial<HarnessOptions<AnyKind[]>> & {
backend?: "memory" | "jsonl";
models?: Models;
dir?: string;
root?: HarnessOptions["root"];
hooks?: HooksByKind;
taskKinds?: AnyKind[];
sections?: SystemSection[];
setup?: (harness: Harness<AnyKind[]>) => void;
},
): Promise<Env> {
const dir = opts.dir ?? (opts.backend === "jsonl" ? mkdtempSync(join(tmpdir(), "v3t-")) : undefined);
const storage: Storage = dir ? await JsonlStorage.open(dir, { fsync: false }) : new MemoryStorage();
const models = opts.models ?? fake({ respond: echoScript });
const h = await Harness.open<AnyKind[]>(
storage,
{
models,
tools: opts.tools,
taskKinds: opts.taskKinds,
sections: opts.sections,
plugins: opts.plugins,
processHost: opts.processHost,
root: opts.root ?? { rewindable: { model, selectedTools: opts.tools?.map((t) => t.name) ?? [] } },
onReport: opts.onReport,
},
ctx,
);
opts.setup?.(h);
const hookNamespace = opts.hooks === undefined ? undefined : h.namespace("test.hooks", {});
if (opts.hooks?.generation) h.hooks(hookNamespace!, builtin.generation, opts.hooks.generation);
if (opts.hooks?.tool) h.hooks(hookNamespace!, builtin.tool, opts.hooks.tool);
if (opts.hooks?.postTools) h.hooks(hookNamespace!, builtin.postTools, opts.hooks.postTools);
if (opts.hooks?.collapse) h.hooks(hookNamespace!, builtin.collapse, opts.hooks.collapse);
h.resume();
const root = await h.root(ctx);
let closed = false;
const env: Env = {
h,
root,
storage,
dir,
entries: async (c = 1) => (await h.entries({ conversationId: c, limit: 1000 }, ctx)).reverse(),
tasks: (c) => root.commit((tx) => tx.tasks(c === undefined ? {} : { conversationId: c }), ctx),
input: (id) => root.commit((tx) => tx.input(id), ctx),
async crash() {
if (!dir) throw new Error("crash needs jsonl");
await h.close(ctx); // signals + joins, writes nothing: same durable state as a kill
return () => open({ ...opts, dir, backend: "jsonl" });
},
async close() {
if (closed) return;
closed = true;
await h.close(ctx);
if (dir && !opts.dir) rmSync(dir, { recursive: true, force: true });
},
};
return env;
}
export async function collectWatch(root: Pick<ConversationHandle<object>, "watch">) {
const envelopes: Envelope[] = [];
const w = await root.watch(ctx);
w.start((d) => envelopes.push(d));
return { view: w.view, envelopes, stop: () => w.stop() };
}
export const phaseOf = (t: Task | undefined) => t?.checkpoint?.phase;
export const liveTasks = async (env: Env, c?: number) => (await env.tasks(c)).filter((t) => t.status !== "terminal");
export async function untilPhase(env: Env, kind: string, phase: string | undefined, timeoutMs = 3000): Promise<Task> {
const t0 = Date.now();
for (;;) {
const t = (await env.tasks()).find((t) => t.kind === kind && t.status !== "terminal" && phaseOf(t) === phase);
if (t) return t;
if (Date.now() - t0 > timeoutMs)
throw new Error(
`timeout waiting for ${kind}@${phase}; live: ${(await liveTasks(env)).map((t) => `${t.kind}@${phaseOf(t)}`).join(",")}`,
);
await sleep(2);
}
}
export async function untilTerminal(env: Env, id: number, timeoutMs = 3000): Promise<Task> {
const t0 = Date.now();
for (;;) {
const t = (await env.tasks()).find((t) => t.id === id);
if (t?.status === "terminal") return t;
if (Date.now() - t0 > timeoutMs) throw new Error(`timeout waiting for task ${id}: ${t?.status}@${phaseOf(t)}`);
await sleep(5);
}
}

View file

@ -0,0 +1,783 @@
import assert from "node:assert/strict";
import type { ToolCall } from "@earendil-works/pi-ai";
import { Type } from "typebox";
import { onTestFinished, test } from "vitest";
import { applyEnvelope, kinds } from "../../../src/harness/pico3/harness.ts";
import type { JsonlStorage } from "../../../src/harness/pico3/jsonl.ts";
import { Session } from "../../../src/harness/pico3/session.ts";
import { defineSystemSection, sectionSeed, systemSections } from "../../../src/harness/pico3/system.ts";
import { defineTask, Forbidden, type RequestMessage, type ToolDeclaration } from "../../../src/harness/pico3/types.ts";
import {
type CoreTx,
collectWatch,
contentOf,
ctx,
echoScript,
failureOf,
fake,
Gate,
kinds as kindsOf,
lastMessage,
model,
open,
sleep,
tool,
untilTerminal,
} from "./helpers.ts";
const resultOf = async (env: Awaited<ReturnType<typeof open>>) =>
(await env.entries()).find((e) => e.kind === "pi.tool_result")!;
const dataOf = (e: { data?: unknown }) =>
e.data as { diagnostics?: { code: string }[]; truncated?: { bytes: number; lines: number }; details?: unknown };
// ---------------------------------------------------------------------------
// Tools
// ---------------------------------------------------------------------------
test("tool not offered → synthetic error, tool never invoked, registry not consulted", async () => {
const t = tool("hidden");
const env = await open({ tools: [t], root: { rewindable: { model, selectedTools: [] } } });
onTestFinished(() => env.close());
await (await env.root.send({ content: "tool:hidden" }, ctx)).wait(ctx);
const r = await resultOf(env);
assert.equal(t.calls, 0);
assert.equal(dataOf(r).diagnostics?.[0]?.code, "not_offered");
assert.match(contentOf(r), /not offered/);
});
test("tool offered but not registered → missing_tool", async () => {
const env = await open({ tools: [], root: { rewindable: { model, selectedTools: ["ghost"] } } });
onTestFinished(() => env.close());
// selectedTools is filtered against the registry at prepare, so "ghost" is never offered → not_offered.
await (await env.root.send({ content: "tool:ghost" }, ctx)).wait(ctx);
assert.equal(dataOf(await resultOf(env)).diagnostics?.[0]?.code, "not_offered");
});
test("invalid arguments → invalid_arguments, not invoked", async () => {
const t = tool("strict");
const models = fake({
respond: (m) =>
lastMessage(m).role === "toolResult"
? { text: "ok" }
: { toolCalls: [{ name: "strict", arguments: { v: 42 } }] },
});
const env = await open({ tools: [t], models });
onTestFinished(() => env.close());
await (await env.root.send({ content: "go" }, ctx)).wait(ctx);
assert.equal(t.calls, 0);
assert.equal(dataOf(await resultOf(env)).diagnostics?.[0]?.code, "invalid_arguments");
});
test("beforeTool: block → blocked; rewrite arguments → invoked with rewritten; identity change → blocked; throw → blocked", async () => {
const cases: [string, (c: ToolCall) => unknown, string, number][] = [
["block", () => ({ block: "no" }), "blocked", 0],
["rewrite", (c) => ({ call: { ...c, arguments: { v: "rewritten" } } }), "", 1],
["identity", (c) => ({ call: { ...c, name: "other" } }), "blocked", 0],
[
"throw",
() => {
throw new Error("hook boom");
},
"blocked",
0,
],
];
for (const [label, hook, code, calls] of cases) {
const t = tool("x");
const env = await open({ tools: [t], hooks: { tool: { beforeTool: hook as never } } });
onTestFinished(() => env.close().catch(() => {}));
await (await env.root.send({ content: "tool:x" }, ctx)).wait(ctx);
const r = await resultOf(env);
assert.equal(t.calls, calls, label);
if (code) assert.equal(dataOf(r).diagnostics?.[0]?.code, code, label);
else assert.match(contentOf(r), /x\(rewritten\)/, label);
await env.close();
}
});
test("afterTool can replace the result; tool throw becomes an error result", async () => {
const t = tool("x", { throws: "kaboom" });
const env = await open({
tools: [t],
hooks: {
tool: {
afterTool: (_c, r) =>
r.isError ? { ...r, content: [{ type: "text", text: "softened" }], isError: false } : undefined,
},
},
});
onTestFinished(() => env.close());
await (await env.root.send({ content: "tool:x" }, ctx)).wait(ctx);
const r = await resultOf(env);
assert.equal(contentOf(r), JSON.stringify([{ type: "text", text: "softened" }]));
assert.equal((r.model![0] as { isError: boolean }).isError, false);
});
test("output bounding: head keeps the first maxBytes, tail the last; lines cap; truncated recorded", async () => {
const big = Array.from({ length: 500 }, (_, i) => `line${i}`).join("\n");
for (const retain of ["head", "tail"] as const) {
const t: ToolDeclaration = {
name: "big",
description: "",
parameters: Type.Object({ v: Type.String() }),
output: { maxBytes: 2000, maxLines: 100, retain },
async execute() {
return { content: [{ type: "text", text: big }] };
},
};
const env = await open({ tools: [t] });
onTestFinished(() => env.close().catch(() => {}));
await (await env.root.send({ content: "tool:big" }, ctx)).wait(ctx);
const r = await resultOf(env);
const text = (r.model![0] as { content: { text: string }[] }).content[0]!.text;
const lines = text.split("\n");
assert.equal(lines.length, 100, retain);
assert.equal(lines[0], retain === "head" ? "line0" : "line400", retain);
assert.ok(dataOf(r).truncated!.lines === 400, retain);
assert.equal(dataOf(r).diagnostics?.[0]?.code, "truncated");
await env.close();
}
});
test("tool slot: turn.tools[index] carries call and bounded output; suffix updates append; cleared when the turn ends", async () => {
const gate = new Gate();
const t: ToolDeclaration = {
name: "p",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute(_a, api, c) {
api.stream("half");
await api.progress((o) => {
o.progress = "50%";
}, c);
await gate.wait(c);
api.stream(" full");
await api.progress((o) => {
o.progress = "100%";
}, c);
return { content: [{ type: "text", text: "done" }] };
},
};
const env = await open({ tools: [t] });
onTestFinished(() => env.close());
const { envelopes, stop } = await collectWatch(env.root);
const a = await env.root.send({ content: "tool:p" }, ctx);
await gate.arrivals(1);
const mid = await env.root.sticky(ctx);
assert.equal(mid.turn.tools.length, 1);
assert.equal(mid.turn.tools[0]!.callId, "call_0_0");
assert.equal(mid.turn.tools[0]!.output, "half");
gate.open();
await a.wait(ctx);
await env.root.waitForIdle(ctx);
assert.deepEqual((await env.root.sticky(ctx)).turn, { tools: [] });
assert.deepEqual((await env.root.sticky(ctx)).tasks, {});
const ops = envelopes.flatMap((envelope) => envelope.ops);
assert.ok(
ops.some((op) => op[0] === "a" && op[1].join(".") === "turn.tools.0.output" && op[2] === " full"),
"the suffix update is an append on the output path",
);
stop();
});
test("ordinary kinds cannot appendEntry or create core tasks (runtime Forbidden), but can write()", async () => {
const seen: string[] = [];
const env = await open({
plugins: {
probe: async (_i, api, c) => {
await api
.task(kinds.job, { command: "x", args: [], cwd: "/", notify: false, rerun: false }, {}, c)
.catch((e) => seen.push(String(e)));
return "ok";
},
},
hooks: {},
processHost: undefined,
});
onTestFinished(() => env.close());
// appendEntry from a non-core invoker
await assert.rejects(
env.root.commit((tx) => (tx as unknown as CoreTx).appendEntry(1, { kind: "pi.notice" }), ctx),
Forbidden,
);
// core task creation from an ordinary task: use pi.plugin to try
const id = await env.root.commit(
(tx) =>
tx.createTask(kinds.plugin, { handler: "probe", input: null }, { conversationId: 1, background: true }).id,
ctx,
);
await untilTerminal(env, id);
// pi.job is ordinary, so that succeeded; now try a core kind directly
const id2 = await env.root
.commit(
(tx) =>
tx.createTask(kinds.plugin, { handler: "core", input: null }, { conversationId: 1, background: true }).id,
ctx,
)
.catch(() => undefined);
void id2;
const env2 = await open({
plugins: {
core: async (_i, api, c) => {
try {
await api.task(kinds.generation, { inputs: [] }, {}, c);
return "created";
} catch (e) {
return String(e);
}
},
},
});
onTestFinished(() => env2.close());
const id3 = await env2.root.commit(
(tx) => tx.createTask(kinds.plugin, { handler: "core", input: null }, { conversationId: 1, background: true }).id,
ctx,
);
const done = await untilTerminal(env2, id3);
assert.match(String((done.outcome as unknown as { result: string }).result), /Forbidden/);
// write() from host is fine
const w = await env.root.write({ kind: "pi.notice", model: [{ role: "user", content: "n", timestamp: 0 }] }, ctx);
assert.equal((await env.input(w))?.status, "done");
});
// ---------------------------------------------------------------------------
// Collapse
// ---------------------------------------------------------------------------
const summarizer = (m: RequestMessage[]) =>
m.some(
(x) => x.role === "user" && String((x as { content: unknown }).content).includes("Respond with the summary only"),
)
? { text: "SUMMARY" }
: echoScript(m);
test("threshold collapse: created foreground after a response over threshold; summary head placed; context shrinks", async () => {
const env = await open({
models: fake({ respond: summarizer }),
root: { rewindable: { model, threshold: 100, keepRecent: 30 } },
});
onTestFinished(() => env.close());
for (const t of ["one", "two", "three", "four"]) await (await env.root.send({ content: t }, ctx)).wait(ctx);
await env.root.waitForIdle(ctx);
const collapses = (await env.tasks()).filter((t) => t.kind === "pi.collapse");
assert.ok(collapses.length >= 1);
assert.equal(collapses[0]!.background, undefined); // foreground
assert.equal((collapses[0]!.input as { reason: string }).reason, "threshold");
const { entries, head } = await env.root.commit((tx) => tx.context(1), ctx);
assert.equal(head?.kind, "pi.summary");
assert.ok(entries.length < 8);
assert.equal(contentOf(head), "SUMMARY ");
// summarizer request was tool-free and ended with the instruction
const req = (env.h as unknown as { options?: unknown }) && null;
void req;
});
test("beforeCollapse: decline → failed/declined; hook summary → no provider call; instructions override", async () => {
for (const mode of ["decline", "summary", "instructions"] as const) {
const models = fake({ respond: summarizer });
const env = await open({
models,
root: { rewindable: { model, keepRecent: 10 } },
hooks: {
collapse: {
beforeCollapse: () =>
mode === "decline"
? { decline: true }
: mode === "summary"
? { summary: "HOOKED" }
: { instructions: "Be terse. Respond with the summary only" },
},
},
});
onTestFinished(() => env.close().catch(() => {}));
for (const t of ["one", "two", "three"]) await (await env.root.send({ content: t }, ctx)).wait(ctx);
const before = models.calls;
const cid = await env.root.collapse(undefined, ctx);
const c = await untilTerminal(env, cid);
if (mode === "decline") {
assert.equal((c.outcome as unknown as { failure: { reason: string } }).failure.reason, "declined");
assert.equal(models.calls, before);
} else if (mode === "summary") {
assert.equal(c.outcome?.status, "completed");
assert.equal(models.calls, before);
assert.equal(contentOf((await env.entries()).find((e) => e.kind === "pi.summary")), "HOOKED");
} else {
assert.equal(c.outcome?.status, "completed");
assert.equal(models.calls, before + 1);
const last = models.requests[models.requests.length - 1]!;
assert.match(String((last[last.length - 1] as { content: string }).content), /Be terse/);
}
await env.close();
}
});
test("collapse is stale if another head lands between snapshot and finalization", async () => {
const gate = new Gate();
const models = fake({ respond: summarizer, gate });
const env = await open({ models, root: { rewindable: { model, keepRecent: 10 } } });
onTestFinished(() => env.close());
gate.open();
for (const t of ["one", "two", "three"]) await (await env.root.send({ content: t }, ctx)).wait(ctx);
gate.close();
const cid = await env.root.collapse(undefined, ctx);
await gate.arrivals(4); // summarizer in flight
await env.root.reset(undefined, ctx); // a new head
gate.open();
const c = await untilTerminal(env, cid);
assert.equal((c.outcome as unknown as { failure: { reason: string } }).failure.reason, "stale");
assert.equal((await env.entries()).filter((e) => e.kind === "pi.summary").length, 0);
});
test("manual collapse with nothing to collapse rejects", async () => {
const env = await open({});
onTestFinished(() => env.close());
await (await env.root.send({ content: "one" }, ctx)).wait(ctx);
await assert.rejects(env.root.collapse(undefined, ctx), /nothing to collapse/);
});
// ---------------------------------------------------------------------------
// Forks and documents
// ---------------------------------------------------------------------------
test("fork: rewindable as of the fork point, sticky defaults, inbox not inherited, context inherited up to `at`", async () => {
const gate = new Gate();
const env = await open({
models: fake({ respond: echoScript, gate }),
root: { rewindable: { model }, sticky: { followUpMode: "all" } },
});
onTestFinished(() => env.close());
const state = env.h.namespace<{ k: string; s: string }>("test.fork-state", {
rewindable: { k: "" },
sticky: { s: "" },
});
gate.open();
await env.root.commit((tx) => {
tx.plugins(state).k = "v1";
}, ctx);
const a = await (await env.root.send({ content: "one" }, ctx)).wait(ctx);
await env.root.commit((tx) => {
const plugins = tx.plugins(state);
plugins.k = "v2";
plugins.s = "sticky";
}, ctx);
await (await env.root.send({ content: "two" }, ctx)).wait(ctx);
gate.close();
const busy = await env.root.send({ content: "three" }, ctx);
await gate.arrivals(3);
await env.root.send({ content: "queued" }, ctx);
const fork = await env.root.fork(a.answer!, {}, ctx);
const fr = await fork.rewindable(ctx);
const fs = await fork.sticky(ctx);
assert.equal(fr.plugins["test.fork-state"]?.k, "v1");
assert.equal(fr.model?.modelId, "fake-1");
assert.equal(fs.followUpMode, "one-at-a-time"); // sticky not inherited
assert.equal(fs.plugins["test.fork-state"]?.s, undefined);
assert.deepEqual(fs.inbox, []);
const { entries } = await fork.commit((tx) => tx.context(fork.id), ctx);
assert.equal(kindsOf(entries), "user system assistant");
// fork is independent: it can run while the parent is busy
gate.open();
const fa = await (await fork.send({ content: "in fork" }, ctx)).wait(ctx);
assert.equal(fa.status, "done");
await busy.wait(ctx);
assert.equal(kindsOf(await env.entries(fork.id)), "user system assistant user assistant"); // fork-aware: inherited prefix + own (system unchanged → not re-emitted)
assert.equal(
kindsOf((await fork.commit((tx) => tx.context(fork.id), ctx)).entries),
"user system assistant user assistant",
);
// parent's later writes are invisible to the fork
assert.equal((await fork.rewindable(ctx)).plugins["test.fork-state"]?.k, "v1");
});
test("fork at 'start' inherits nothing", async () => {
const env = await open({});
onTestFinished(() => env.close());
await (await env.root.send({ content: "one" }, ctx)).wait(ctx);
const fork = await env.root.fork("start", { rewindable: { model } }, ctx);
assert.equal(kindsOf((await fork.commit((tx) => tx.context(fork.id), ctx)).entries), "");
assert.equal((await fork.commit((tx) => tx.conversation(fork.id), ctx))?.parent, undefined);
});
test("sticky sidecar truncates to one base after retire; rewindable log keeps history", async () => {
const env = await open({ backend: "jsonl" });
onTestFinished(() => env.close());
const sizes = () => (env.storage as JsonlStorage).sizes();
for (let i = 0; i < 5; i++) await (await env.root.send({ content: "x ".repeat(200) }, ctx)).wait(ctx);
await env.root.waitForIdle(ctx);
await sleep(20);
const s1 = sizes()["sticky-1.jsonl"]!;
for (let i = 0; i < 5; i++) await (await env.root.send({ content: "x ".repeat(200) }, ctx)).wait(ctx);
await env.root.waitForIdle(ctx);
await sleep(20);
const s2 = sizes()["sticky-1.jsonl"]!;
assert.ok(Math.abs(s1 - s2) < 200, `sticky steady state: ${s1} vs ${s2}`);
assert.ok(sizes()["main.jsonl"]! > 10_000);
});
test("watch: capture + stream == fresh snapshot after a full turn; head truncates view.entries", async () => {
const env = await open({
tools: [tool("a")],
root: { rewindable: { model, selectedTools: ["a"], keepRecent: 10 } },
models: fake({ respond: summarizer }),
});
onTestFinished(() => env.close());
const state = env.h.namespace<{ w: number }>("test.watch-private", { rewindable: { w: 0 } });
const { view, envelopes, stop } = await collectWatch(env.root);
await (await env.root.send({ content: "tool:a" }, ctx)).wait(ctx);
await env.root.commit((tx) => {
tx.plugins(state).w = 1;
}, ctx);
const cid = await env.root.collapse(undefined, ctx);
await untilTerminal(env, cid);
await env.root.waitForIdle(ctx);
await sleep(20);
stop();
let folded = view;
for (const d of envelopes) folded = applyEnvelope(folded, d);
const fresh = await collectWatch(env.root);
fresh.stop();
assert.deepEqual(
folded.entries.map((e) => e.id),
fresh.view.entries.map((e) => e.id),
);
assert.deepEqual(folded, fresh.view);
assert.deepEqual(folded.tasks, {});
// the summary head truncated the view
const summary = folded.entries.find((e) => e.kind === "pi.summary")!;
assert.ok(folded.entries.every((e) => e.id >= summary.head!));
});
test("plugin task: handler runs with the api, result recorded; missing handler fails", async () => {
const env = await open({ plugins: { double: async (i) => (i as number) * 2 } });
onTestFinished(() => env.close());
const id = await env.root.commit(
(tx) => tx.createTask(kinds.plugin, { handler: "double", input: 21 }, { conversationId: 1, background: true }).id,
ctx,
);
assert.deepEqual((await untilTerminal(env, id)).outcome, { status: "completed", result: 42 });
const id2 = await env.root.commit(
(tx) => tx.createTask(kinds.plugin, { handler: "nope", input: null }, { conversationId: 1, background: true }).id,
ctx,
);
assert.equal(failureOf(await untilTerminal(env, id2))?.reason, "missing_handler");
});
test("system sections: unchanged → no new system entry; changed → delta; after a head → full baseline with omission edits", async () => {
let prompt = "P1";
const env = await open({
hooks: {
generation: {
systemInstructions: ({ sections }) => {
sections.set(systemSections.identity, prompt);
},
},
},
models: fake({ respond: summarizer }),
root: { rewindable: { model, keepRecent: 10 } },
});
onTestFinished(() => env.close());
await (await env.root.send({ content: "one" }, ctx)).wait(ctx);
await (await env.root.send({ content: "two" }, ctx)).wait(ctx);
assert.equal((await env.entries()).filter((e) => e.kind === "pi.system").length, 1);
prompt = "P2";
await (await env.root.send({ content: "three" }, ctx)).wait(ctx);
const sys = (await env.entries()).filter((e) => e.kind === "pi.system");
assert.equal(sys.length, 2);
assert.equal((sys[0]!.data as { baseline?: true }).baseline, true); // first ever: baseline (no head, no prior managed entry → §12.4 says baseline when newestBaseline is null and a head exists; here it's the seed case)
assert.equal((sys[1]!.data as { baseline?: true; sections: { key: string; action: string }[] }).baseline, undefined); // delta
assert.deepEqual(
(sys[1]!.data as { sections: { key: string; action: string }[] }).sections.map((r) => `${r.action}:${r.key}`),
["set:identity"],
);
assert.match((sys[1]!.model![0] as { content: string }).content, /identity section now reads:\nP2/);
await env.root.reset(undefined, ctx);
await (await env.root.send({ content: "four" }, ctx)).wait(ctx);
const sys2 = (await env.entries()).filter((e) => e.kind === "pi.system");
assert.equal(sys2.length, 3);
assert.equal((sys2[2]!.data as { baseline?: true }).baseline, true);
assert.match((sys2[2]!.model![0] as { content: string }).content, /## identity\nP2/);
const { messages } = await env.root.commit((tx) => tx.context(1), ctx);
assert.equal(messages.filter((m) => m.role === "system").length, 1); // earlier managed entries are before the head or omitted
});
test("context: a tool result missing after a fork is synthesised", async () => {
const env = await open({ tools: [tool("a")] });
onTestFinished(() => env.close());
await (await env.root.send({ content: "tool:a" }, ctx)).wait(ctx);
const entries = await env.entries();
const assistant = entries.find((e) => e.kind === "pi.assistant")!;
// fork at the assistant with tool calls: its result is after `at` and invisible
const fork = await env.root.fork(assistant.id, {}, ctx);
const { messages } = await fork.commit((tx) => tx.context(fork.id), ctx);
const tr = messages.find((message) => message.role === "toolResult");
assert.ok(tr?.role === "toolResult");
assert.equal(tr.isError, true);
assert.equal((tr.details as { reason?: string } | undefined)?.reason, "missing_after_fork");
});
test("api.stream: kernel bounds per declaration, throttles flushes, result content defaults to the bounded stream", async () => {
const t: ToolDeclaration = {
name: "s",
description: "",
parameters: Type.Object({ v: Type.String() }),
output: { maxBytes: 100, retain: "tail" },
async execute(_a, api) {
for (let i = 0; i < 50; i++) api.stream(`chunk${i} `);
return {};
},
};
const env = await open({ tools: [t] });
onTestFinished(() => env.close());
const { envelopes, stop } = await collectWatch(env.root);
await (await env.root.send({ content: "tool:s" }, ctx)).wait(ctx);
stop();
const r = await resultOf(env);
const text = (r.model![0] as { content: { text: string }[] }).content[0]!.text;
assert.ok(text.length <= 100 && text.endsWith("chunk49 "), text);
assert.equal(dataOf(r).diagnostics?.[0]?.code, "truncated");
const flushes = envelopes.filter((envelope) =>
envelope.ops.some((op) => op[0] === "s" && op[1].includes("output")),
).length;
assert.ok(flushes <= 2, `throttled: ${flushes} flushes for 50 chunks`);
});
test("tools cannot bound their own output: a returned content larger than the bound is still bounded by the kernel", async () => {
const t: ToolDeclaration = {
name: "big",
description: "",
parameters: Type.Object({ v: Type.String() }),
output: { maxBytes: 10 },
async execute() {
return { content: [{ type: "text", text: "x".repeat(1000) }] };
},
};
const env = await open({ tools: [t] });
onTestFinished(() => env.close());
await (await env.root.send({ content: "tool:big" }, ctx)).wait(ctx);
const r = await resultOf(env);
assert.equal((r.model![0] as { content: { text: string }[] }).content[0]!.text.length, 10);
assert.equal(dataOf(r).truncated!.bytes, 990);
});
test("§12: sections render through the draft; wrap applies after render; delete emits a remove; a throwing handler's edits roll back", async () => {
const custom = defineSystemSection<{ n: number }>({ key: "custom", render: (v) => `custom=${v.n}` });
let throwOnce = true;
const env = await open({
sections: [custom],
hooks: {
generation: {
systemInstructions: ({ sections }) => {
sections.set(systemSections.identity, "I am");
sections.set(custom, { n: 1 });
sections.wrap(systemSections.identity, (r) => `[${r}]`);
},
},
},
models: fake({ respond: summarizer }),
root: { rewindable: { model, keepRecent: 10 } },
});
onTestFinished(() => env.close());
const hookNamespace = env.h.namespace("test.section-hooks", {});
env.h.hooks(hookNamespace, kinds.generation, {
systemInstructions: ({ sections }) => {
if (throwOnce) {
throwOnce = false;
sections.set(custom, { n: 999 });
throw new Error("boom");
}
},
});
await (await env.root.send({ content: "one" }, ctx)).wait(ctx);
const sys = (await env.entries()).filter((e) => e.kind === "pi.system");
const content = (sys[0]!.model![0] as { content: string }).content;
assert.match(content, /## identity\n\[I am\]/);
assert.match(content, /## custom\ncustom=1/); // the throwing handler's n=999 was rolled back
const data = sys[0]!.data as { sections: { key: string; action: string; value?: unknown }[] };
assert.deepEqual(
data.sections.map((s) => s.key),
["identity", "custom"],
);
// delete → remove delta
env.h.hooks(hookNamespace, kinds.generation, {
systemInstructions: ({ sections }) => {
sections.delete(custom);
},
});
await (await env.root.send({ content: "two" }, ctx)).wait(ctx);
const sys2 = (await env.entries()).filter((e) => e.kind === "pi.system");
assert.equal(sys2.length, 2);
assert.deepEqual((sys2[1]!.data as { sections: { key: string; action: string }[] }).sections, [
{ key: "custom", action: "remove" },
]);
assert.match((sys2[1]!.model![0] as { content: string }).content, /custom section no longer applies/);
});
test("§12.5: a config change while systemInstructions runs makes the preparation retry with fresh inputs", async () => {
const gate = new Gate();
let runs = 0;
const env = await open({
hooks: {
generation: {
systemInstructions: async ({ sections, config }, _i, c) => {
runs++;
sections.set(systemSections.identity, `profile=${config.profile}`);
if (runs === 1) await gate.wait(c);
},
},
},
});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "one" }, ctx);
await gate.arrivals(1);
await env.root.config.set({ profile: "changed" }, ctx); // snapshot S is now stale
gate.open();
await a.wait(ctx);
assert.equal(runs, 2); // discarded and repeated
const sys = (await env.entries()).find((e) => e.kind === "pi.system")!;
assert.match((sys.model![0] as { content: string }).content, /profile=changed/);
});
test("§8.1: a section seed applies to a new conversation while it has no managed entry", async () => {
const env = await open({});
onTestFinished(() => env.close());
const child = await env.h.createConversation(
{ rewindable: { model }, sections: [sectionSeed(systemSections.identity, "seeded identity")] },
ctx,
);
await (await child.send({ content: "hi" }, ctx)).wait(ctx);
const sys = (await env.entries(child.id)).find((e) => e.kind === "pi.system")!;
assert.match((sys.model![0] as { content: string }).content, /## identity\nseeded identity/);
});
test("c.config: flat get/set across both documents; typed by registered kinds; disjointness enforced at open", async () => {
const plan = defineTask<null, { phase: "x" }, null, null, null, object, { rewindable: { planMode: boolean } }>({
name: "plan",
config: { rewindable: { planMode: false } },
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async initial() {
return { next: { phase: "x" } };
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [plan] });
onTestFinished(() => env.close());
const before = await env.root.config.get(ctx);
assert.equal((before as { planMode: boolean }).planMode, false); // default from the kind
assert.equal(before.followUpMode, "one-at-a-time");
await env.root.config.set({ planMode: true, followUpMode: "all", keepRecent: 5 } as never, ctx);
assert.equal((await env.root.rewindable(ctx)).planMode, true);
assert.equal((await env.root.rewindable(ctx)).keepRecent, 5);
assert.equal((await env.root.sticky(ctx)).followUpMode, "all");
await env.root.config.reset(["planMode", "followUpMode", "keepRecent"], ctx);
const reset = await env.root.config.get(ctx);
assert.equal((reset as { planMode: boolean }).planMode, false);
assert.equal(reset.followUpMode, "one-at-a-time");
assert.equal(reset.keepRecent, 20_000);
await assert.rejects(env.root.config.set({ keepRecent: undefined } as never, ctx), /use config\.reset/);
const clash = defineTask<null, { phase: "x" }, null, null, null, object, { sticky: { model: string } }>({
name: "clash",
config: { sticky: { model: "" } },
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async initial() {
return { next: { phase: "x" } };
},
async abort() {
return () => null;
},
});
await assert.rejects(open({ taskKinds: [clash] as never }), /declared by more than one kind/);
});
test("defineTask: an ordinary kind with phases runs, checkpoints, recovers into its phase, and is typed via TaskRef", async () => {
const gate = new Gate();
const ran: string[] = [];
const counter = defineTask<{ start: number }, { phase: "counting"; n: number }, { total: number }, never, null>({
name: "counter",
async initial(task) {
ran.push("initial");
return { next: { phase: "counting", n: task.input.start } };
},
phases: {
async counting(task, _rt, ctx) {
ran.push(`counting@${task.checkpoint.n}`);
if (task.checkpoint.n < 3) {
await gate.wait(ctx);
return { next: { phase: "counting", n: task.checkpoint.n + 1 } };
}
return { done: () => ({ status: "completed", result: { total: task.checkpoint.n } }) };
},
},
async abort() {
return () => null;
},
});
let env = await open({ backend: "jsonl", taskKinds: [counter] });
onTestFinished(() => env.close());
const ref = await env.root.commit(
(tx) => tx.createTask(counter, { start: 1 }, { conversationId: 1, background: true }),
ctx,
);
await gate.arrivals(1);
await env.crash();
env = await open({ dir: env.dir, backend: "jsonl", taskKinds: [counter] });
gate.open();
const done = await untilTerminal(env, ref.id);
assert.deepEqual(done.outcome, { status: "completed", result: { total: 3 } });
assert.deepEqual(ran, ["initial", "counting@1", "counting@1", "counting@2", "counting@3"]); // resumed into `counting@1` after the crash, not `initial`
});
test("sticky log: a base is written when the conversation goes idle, or when ops exceed the budget while busy — never per task", async () => {
const gate = new Gate();
// A tool that streams a lot while another tool holds the turn open.
const noisy: ToolDeclaration = {
name: "noisy",
description: "",
parameters: Type.Object({ v: Type.String() }),
output: { maxBytes: 100_000, retain: "tail" },
async execute(_a, api, c) {
for (let i = 0; i < 40; i++) {
api.stream(`${i}:`.repeat(2_000));
await sleep(12);
}
await gate.wait(c);
return {};
},
};
const env = await open({
backend: "jsonl",
tools: [noisy],
models: fake({
respond: (m) =>
lastMessage(m).role === "toolResult"
? { text: "ok" }
: {
toolCalls: [
{ name: "noisy", arguments: { v: "a" } },
{ name: "noisy", arguments: { v: "b" } },
],
},
}),
});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "go" }, ctx);
await gate.arrivals(2);
const bytesWhileBusy = (env.storage as JsonlStorage).sizes()["sticky-1.jsonl"]!;
assert.ok(bytesWhileBusy > Session.STICKY_BASE_BUDGET, `streamed ${bytesWhileBusy} bytes of sticky state`);
gate.open();
await a.wait(ctx);
await env.root.waitForIdle(ctx);
await sleep(20);
const sizes = (env.storage as JsonlStorage).sizes();
assert.ok(sizes["sticky-1.jsonl"]! < 5_000, `truncated to one small base: ${sizes["sticky-1.jsonl"]}`);
});

View file

@ -0,0 +1,169 @@
import assert from "node:assert/strict";
import { onTestFinished, test } from "vitest";
import { Membrane } from "../../../src/harness/pico3/membrane.ts";
import { defineTask, type JsonValue, type Namespace } from "../../../src/harness/pico3/types.ts";
import { ctx, open } from "./helpers.ts";
test("membrane unit: root and nested wrappers share one liveness flag; identity preserved; wrappers cannot be assigned into the doc", () => {
const target = { a: { b: [1, 2] } };
const m = new Membrane("t");
const root = m.wrap(target);
const nested = root.a;
const arr = root.a.b;
assert.equal(root.a, nested); // WeakMap identity
assert.notEqual(nested, target.a); // never the raw object
arr.push(3);
assert.deepEqual(target.a.b, [1, 2, 3]); // forwarded
assert.throws(() => {
(root as { c?: unknown }).c = nested;
}, /assigning a document proxy/);
assert.deepEqual(Object.keys(root), ["a"]);
assert.notEqual(Object.getOwnPropertyDescriptor(root, "a")!.value, target.a); // descriptors do not leak raw
const assigned = { value: 1 };
Object.defineProperty(root, "defined", { value: assigned, enumerable: true, writable: true, configurable: true });
assigned.value = 2;
assert.deepEqual((target as { defined?: { value: number } }).defined, { value: 1 });
assert.throws(() => Object.defineProperty(root, "accessor", { get: () => 1 }), /cannot define accessors/);
assert.throws(() => Object.setPrototypeOf(root, null), /cannot change prototypes/);
assert.throws(() => Object.preventExtensions(root), /cannot change extensibility/);
m.revoke();
assert.throws(() => root.a, TypeError);
assert.throws(() => nested.b, TypeError);
assert.throws(() => arr.length, TypeError);
assert.throws(() => arr.push(4), TypeError);
assert.throws(() => {
nested.b = [];
}, TypeError);
assert.throws(() => Object.keys(nested), TypeError);
assert.throws(() => "b" in nested, TypeError);
assert.throws(() => Object.isExtensible(nested), TypeError);
assert.deepEqual(target.a.b, [1, 2, 3]); // nothing changed after revoke
});
test("escaped root and nested handles throw after a successful commit, and their attempted mutations are not persisted by a later commit", async () => {
const env = await open({});
onTestFinished(() => env.close());
const state = env.h.namespace<{ list: JsonValue[]; unrelated: boolean }>("test.membrane-success", {
rewindable: { list: [] },
sticky: { unrelated: false },
});
let root!: { [k: string]: JsonValue | undefined };
let nested!: JsonValue[];
await env.root.commit((tx) => {
root = tx.plugins(state);
root.list = [1];
nested = root.list as JsonValue[];
}, ctx);
assert.throws(() => {
root.x = 1;
}, /outside its transaction/);
assert.throws(() => nested.push(2), /outside its transaction/);
assert.throws(() => root.list, /outside its transaction/);
await env.root.commit((tx) => {
tx.plugins(state).unrelated = true;
}, ctx); // a later unrelated commit
const r = await env.root.rewindable(ctx);
assert.deepEqual(r.plugins["test.membrane-success"], { list: [1] });
});
test("array callbacks receive membrane wrappers rather than raw tracked children", async () => {
const env = await open({});
onTestFinished(() => env.close());
const state = env.h.namespace<{ list: { mode: string }[]; unrelated: boolean }>("test.membrane-callback", {
rewindable: { list: [] },
sticky: { unrelated: false },
});
const escaped: { mode: string }[] = [];
await env.root.commit((tx) => {
const list = tx.plugins(state).list;
list.push({ mode: "safe" });
list.forEach((item) => {
escaped.push(item);
});
list.map((item) => escaped.push(item));
list.find((item) => {
escaped.push(item);
return true;
});
list.sort((a, b) => {
escaped.push(a, b);
return 0;
});
}, ctx);
for (const item of escaped)
assert.throws(() => {
item.mode = "escaped";
}, /outside its transaction/);
await env.root.commit((tx) => {
tx.plugins(state).unrelated = true;
}, ctx);
assert.deepEqual((await env.root.rewindable(ctx)).plugins["test.membrane-callback"], {
list: [{ mode: "safe" }],
});
});
test("escaped handles throw after a failed callback too, and the failed transaction's writes are gone", async () => {
const env = await open({});
onTestFinished(() => env.close());
const state = env.h.namespace<{ obj: { [key: string]: JsonValue }; unrelated: boolean }>("test.membrane-failure", {
rewindable: { obj: {} },
sticky: { unrelated: false },
});
let root!: { [k: string]: JsonValue | undefined };
let nested!: { [k: string]: JsonValue | undefined };
await assert.rejects(
env.root.commit((tx) => {
root = tx.plugins(state);
root.obj = { k: 1 };
nested = root.obj as { [k: string]: JsonValue | undefined };
throw new Error("boom");
}, ctx),
/boom/,
);
assert.throws(() => {
nested.k = 2;
}, /outside its transaction/);
assert.throws(() => {
root.obj = null;
}, /outside its transaction/);
await env.root.commit((tx) => {
tx.plugins(state).unrelated = true;
}, ctx);
assert.deepEqual((await env.root.rewindable(ctx)).plugins["test.membrane-failure"], { obj: {} });
});
test("a task runtime's document handle is revoked when its commit ends", async () => {
let escaped: { [k: string]: JsonValue | undefined } | undefined;
let state!: Namespace<{ v: number }>;
let threw: unknown;
const k = defineTask<null, { phase: "x" }, null, null, null>({
name: "esc",
async initial(_t, rt, c) {
await rt.commit((tx) => {
escaped = tx.plugins(state);
escaped.v = 1;
}, c);
try {
escaped!.v = 2;
} catch (e) {
threw = e;
}
return { done: () => ({ status: "completed", result: null }) };
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [k] });
onTestFinished(() => env.close());
state = env.h.namespace("test.membrane-runtime", { sticky: { v: 0 } });
const ref = await env.root.commit((tx) => tx.createTask(k, null, { conversationId: 1, background: true }), ctx);
await env.h.waitForTask(ref.id, ctx);
assert.ok(threw instanceof TypeError);
assert.equal((await env.root.sticky(ctx)).plugins["test.membrane-runtime"]?.v, 1);
});

View file

@ -0,0 +1,335 @@
import assert from "node:assert/strict";
import { withAbortSignal } from "@earendil-works/chord/context";
import { onTestFinished, test } from "vitest";
import { Harness, kinds } from "../../../src/harness/pico3/harness.ts";
import { MemoryStorage } from "../../../src/harness/pico3/memory.ts";
import {
type CoreTx,
defineTask,
Faulted,
type JsonObject,
ReadAfterWrite,
type Runtime,
type Storage,
} from "../../../src/harness/pico3/types.ts";
import { ctx, echoScript, fake, Gate, open } from "./helpers.ts";
/** Runs `fn` with core authority inside a generation's terminal closure? No: through a probe kind and a CoreTx cast, so reads/writes are the real ones. */
function coreProbe(
name: string,
body: (tx: CoreTx, rt: Runtime, cur: { conversationId: number; id: number }) => Promise<unknown>,
out: { value?: unknown; error?: unknown },
) {
return defineTask<null, { phase: "x" }, null, null, null>({
name,
async initial(_t, rt, c) {
try {
out.value = await rt.commit((tx, cur) => body(tx as unknown as CoreTx, rt, cur), c);
} catch (e) {
out.error = e;
}
return { next: { phase: "x" } };
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
}
test("read-your-writes: entry(id), task(ref) with later patches, conversation(id), input by request key all see same-batch writes", async () => {
const env = await open({});
onTestFinished(() => env.close());
const seen = await env.root.commit(async (tx) => {
const w = await tx.write(1, { kind: "note", data: { n: 1 } }); // idle → appended immediately, input done
const input = await tx.input(w);
const entry = await tx.entry(input!.entry!);
const ref = tx.createTask(
kinds.plugin,
{ handler: "none", input: null },
{ conversationId: 1, background: true },
);
const task = await tx.task(ref);
const conv = tx.createConversation({});
const c = await tx.conversation(conv);
return { inputStatus: input?.status, entryData: entry?.data, taskStatus: task?.status, convOk: c?.id === conv };
}, ctx);
assert.deepEqual(seen, { inputStatus: "done", entryData: { n: 1 }, taskStatus: "pending", convOk: true });
});
test("ReadAfterWrite: a scan after an append rejects, poisons the transaction even if caught, and nothing is persisted", async () => {
const env = await open({});
onTestFinished(() => env.close());
const before = (await env.entries()).length;
let caught: unknown;
await assert.rejects(
env.root.commit(async (tx) => {
await tx.write(1, { kind: "note" });
try {
await tx.newestEntry(1);
} catch (e) {
caught = e;
}
return "reached the end";
}, ctx),
ReadAfterWrite,
);
assert.ok(caught instanceof ReadAfterWrite);
assert.equal((await env.entries()).length, before);
// task scans after a task write; entry scans after a write to that conversation only
await assert.rejects(
env.root.commit(async (tx) => {
tx.createTask(kinds.plugin, { handler: "x", input: null }, { conversationId: 1, background: true });
await tx.tasks({});
}, ctx),
ReadAfterWrite,
);
const other = (await env.h.createConversation({}, ctx)).id;
await env.root.commit(async (tx) => {
await tx.write(1, { kind: "note" });
await tx.newestEntry(other);
}, ctx); // a different conversation's scan is fine
await assert.rejects(
env.root.commit(async (tx) => {
await tx.write(1, { kind: "note" });
await tx.context(1);
}, ctx),
ReadAfterWrite,
);
await assert.rejects(
env.root.commit(async (tx) => {
await tx.write(1, { kind: "note" });
await tx.scanEntries({ conversationId: 1, limit: 5 });
}, ctx),
ReadAfterWrite,
);
});
test("ReadAfterWrite from core code: a terminal closure that scanned after appending would be caught (probe)", async () => {
const out: { error?: unknown } = {};
const k = coreProbe(
"scan-after",
async (tx, _rt, cur) => {
await tx.write(cur.conversationId, { kind: "note" });
await tx.newestEntry(cur.conversationId, { withHead: true });
},
out,
);
const env = await open({ taskKinds: [k] });
onTestFinished(() => env.close());
const ref = await env.root.commit((tx) => tx.createTask(k, null, { conversationId: 1, background: true }), ctx);
await env.h.waitForTask(ref.id, ctx);
assert.ok(out.error instanceof ReadAfterWrite);
});
test("config: defaults come from kind declarations; stored null is a value; unset restores the default; custom defaults are visible to c.config and to the task before any set", async () => {
let seenByTask: unknown;
const k = defineTask<
null,
{ phase: "x" },
null,
null,
null,
object,
{ rewindable: { planMode: boolean; note: string | null } }
>({
name: "plan",
config: { rewindable: { planMode: false, note: "n" } },
async initial(_t, rt, c) {
seenByTask = await rt.commit(
(tx, cur) => [
tx.config(cur.conversationId).get("planMode"),
tx.snapshot({ doc: "rewindable", conversationId: cur.conversationId }).planMode,
],
c,
);
return { next: { phase: "x" } };
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [k] });
onTestFinished(() => env.close());
const cfg = env.root.config as unknown as {
get(c: typeof ctx): Promise<Record<string, unknown>>;
set(p: Record<string, unknown>, c: typeof ctx): Promise<void>;
reset(keys: string[], c: typeof ctx): Promise<void>;
};
const all = await cfg.get(ctx);
assert.equal(all.planMode, false);
assert.equal(all.note, "n");
assert.deepEqual(all.model, { provider: "anthropic", modelId: "fake-1" }); // set by the test harness root config; the only optional built-in
assert.equal(all.threshold, 0);
const ref = await env.root.commit((tx) => tx.createTask(k, null, { conversationId: 1, background: true }), ctx);
await env.h.waitForTask(ref.id, ctx);
assert.deepEqual(seenByTask, [false, false]); // both the facade and the document read see the default
await cfg.set({ note: null }, ctx);
assert.equal((await cfg.get(ctx)).note, null); // null stored, not "absent"
await cfg.reset(["note"], ctx);
assert.equal((await cfg.get(ctx)).note, "n"); // deleted → default
await assert.rejects(cfg.set({ bogus: 1 }, ctx), /unknown config key/);
// a rewound fork sees the defaults too
const fork = await env.root.fork("start", {}, ctx);
assert.equal(((await fork.config.get(ctx)) as Record<string, unknown>).planMode, false);
});
test("config: key collisions across kinds are rejected at installation", async () => {
const clash = defineTask<null, { phase: "x" }, null, null, null, object, { sticky: { threshold: number } }>({
name: "clash",
config: { sticky: { threshold: 1 } },
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async initial() {
return { next: { phase: "x" } };
},
async abort() {
return () => null;
},
});
await assert.rejects(
Harness.open(new MemoryStorage(), { models: fake({ respond: echoScript }), taskKinds: [clash] } as never, ctx),
/declared by more than one kind/,
);
const partial = defineTask<null, { phase: "x" }, null, null, null>({
...clash,
name: "partial-clash",
config: { rewindable: { partialKey: 1 }, sticky: { threshold: 1 } },
});
const env = await open({});
onTestFinished(() => env.close());
assert.throws(() => env.h.registerTaskKind(partial), /declared by more than one kind/);
assert.equal("partialKey" in ((await env.root.config.get(ctx)) as Record<string, unknown>), false);
const owner = defineTask<null, { phase: "x" }, null, null, null>({
...clash,
name: "partial-owner",
config: { rewindable: { partialKey: 2 } },
});
const unregister = env.h.registerTaskKind(owner);
assert.equal(((await env.root.config.get(ctx)) as Record<string, unknown>).partialKey, 2);
unregister();
});
test("plugin state: registered namespace slices cannot see or clobber each other", async () => {
const env = await open({});
onTestFinished(() => env.close());
const a = env.h.namespace<{ count: number }>("a", { rewindable: { count: 0 } });
const b = env.h.namespace<{ count: number }>("b", { rewindable: { count: 10 } });
assert.throws(() => env.h.namespace("pi.x", {}), /invalid namespace/);
await env.root.commit((tx) => {
tx.plugins(a).count++;
tx.plugins(b).count++;
}, ctx);
await assert.rejects(
env.root.commit((tx) => {
(tx.plugins(a) as JsonObject).typo = true;
}, ctx),
TypeError,
);
const rewindable = await env.root.rewindable(ctx);
assert.deepEqual(rewindable.plugins, { a: { count: 1 }, b: { count: 11 } });
});
test("ordinary tasks can directly read entries inherited by their fork conversation", async () => {
let observed: number[] = [];
const reader = defineTask<{ entry: number }, { phase: "never" }, null, null, null>({
name: "fork-entry-reader",
async initial(task, runtime, context) {
observed = await runtime.commit(async (tx) => {
const direct = await tx.entry(task.input.entry);
const many = await tx.entries([task.input.entry]);
return [direct?.id, many.get(task.input.entry)?.id].filter((id): id is number => id !== undefined);
}, context);
return { done: () => ({ status: "completed", result: null }) };
},
phases: {
async never() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [reader] });
onTestFinished(() => env.close());
const result = await (await env.root.send({ content: "parent" }, ctx)).wait(ctx);
const inherited = (await env.entries()).find((entry) => entry.kind === "pi.user")!;
const fork = await env.root.fork(result.answer!, {}, ctx);
const ref = await fork.commit(
(tx) => tx.createTask(reader, { entry: inherited.id }, { conversationId: fork.id, background: true }),
ctx,
);
await env.h.waitForTask(ref.id, ctx);
assert.deepEqual(observed, [inherited.id, inherited.id]);
});
test("session line: a storage commit failure faults the session; later operations reject Faulted; a callback failure before persistence does not fault", async () => {
class Flaky extends MemoryStorage {
fail = false;
override async commit(w: Parameters<Storage["commit"]>[0], c: Parameters<Storage["commit"]>[1]) {
if (this.fail) throw new Error("disk full");
return super.commit(w, c);
}
}
const storage = new Flaky();
const h = await Harness.open(storage, { models: fake({ respond: echoScript }) }, ctx);
onTestFinished(() => h.close(ctx).catch(() => {}));
h.resume();
const root = await h.root(ctx);
await assert.rejects(
root.commit(() => {
throw new Error("mine");
}, ctx),
/mine/,
);
await root.write({ kind: "note" }, ctx); // still fine
storage.fail = true;
await assert.rejects(root.write({ kind: "note" }, ctx), Faulted);
await assert.rejects(root.write({ kind: "note" }, ctx), Faulted);
await assert.rejects(root.rewindable(ctx), Faulted);
});
test("session line: listener throws never reach the writer; nested line entry rejects; cancellation is checked before the callback", async () => {
const reports: unknown[] = [];
const env = await open({ onReport: (e) => reports.push(e) });
onTestFinished(() => env.close());
const w = await env.root.watch(ctx);
w.start(() => {
throw new Error("listener boom");
});
await env.root.write({ kind: "note" }, ctx); // resolves
assert.equal(w.closed, true);
assert.match(String(reports[0]), /listener boom/);
await assert.rejects(
env.root.commit((tx, lineCtx) => env.root.commit(() => tx.input(1), lineCtx), ctx),
/nested line operation/,
);
const ac = new AbortController();
const cctx = withAbortSignal(ac.signal, ctx);
ac.abort(new Error("cancelled"));
await assert.rejects(env.root.write({ kind: "note" }, cctx), /cancelled/);
const gate = new Gate();
const p = env.root.commit(async () => {
await gate.wait(ctx);
}, ctx);
const ac2 = new AbortController();
const cctx2 = withAbortSignal(ac2.signal, ctx);
const queued = env.root.write({ kind: "note" }, cctx2); // queued behind p
ac2.abort(new Error("late cancel"));
gate.open();
await p;
await assert.rejects(queued, /late cancel/); // checked again when it reaches the line
});

View file

@ -0,0 +1,309 @@
import assert from "node:assert/strict";
import type { Context } from "@earendil-works/chord";
import { onTestFinished, test } from "vitest";
import { kinds as kindsOf } from "../../../src/harness/pico3/harness.ts";
import type { RequestMessage } from "../../../src/harness/pico3/types.ts";
import {
contentOf,
ctx,
echoScript,
fake,
fakeHost,
Gate,
kinds,
model,
open,
sleep,
tool,
untilPhase,
untilTerminal,
} from "./helpers.ts";
// Every test here: run until a specific durable phase, crash (close without writing),
// reopen on the same JSONL dir, resume, assert the turn completes correctly.
test("crash in generation@requesting → pi.usage{interrupted}, retry, complete", async () => {
const gate = new Gate();
let env = await open({ backend: "jsonl", models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A", requestId: "r" }, ctx);
await untilPhase(env, "pi.generation", "requesting");
await gate.arrivals(1);
await env.crash();
const models2 = fake({ respond: echoScript });
env = await open({ dir: env.dir, backend: "jsonl", models: models2 });
// The requestId still resolves to the original input across the reopen.
const same = await env.root.send({ content: "dup", requestId: "r" }, ctx);
assert.equal(same.id, a.id);
const r = await same.wait(ctx);
assert.equal(r.status, "done");
assert.equal(kinds(await env.entries()), "user system usage assistant");
assert.deepEqual((await env.entries())[2]!.data, { attempt: 1, error: "interrupted" });
assert.equal(models2.calls, 1);
});
test("crash in generation@prepared (blocked in beforeRequest) → hooks rerun, no duplicate system entry", async () => {
const gate = new Gate();
let hookCalls = 0;
const hooks = {
generation: {
beforeRequest: async (_req: unknown, _i: unknown, c: Context) => {
hookCalls++;
await gate.wait(c);
return undefined;
},
},
};
let env = await open({ backend: "jsonl", hooks });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
await untilPhase(env, "pi.generation", "prepared");
await gate.arrivals(1);
await env.crash();
gate.open();
env = await open({ dir: env.dir, backend: "jsonl", hooks });
await env.root.waitForIdle(ctx);
assert.equal(hookCalls, 2); // rerun on recovery
const entries = await env.entries();
assert.equal(entries.filter((e) => e.kind === "pi.system").length, 1); // step 1 never reruns
assert.equal(entries.filter((e) => e.kind === "pi.usage").length, 0); // prepared is not an in-flight effect: no failed attempt recorded
assert.equal((await env.input(a.id))?.status, "done");
});
test("crash in generation@retrying → sleeps out untilMs, then retries", async () => {
let n = 0;
const models = fake({ respond: (m) => (n++ === 0 ? { error: "overloaded" } : echoScript(m)) });
let env = await open({
backend: "jsonl",
models,
root: { rewindable: { model }, sticky: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 300 } } },
});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
const t = await untilPhase(env, "pi.generation", "retrying");
const untilMs = (t.checkpoint as unknown as { untilMs: number }).untilMs;
await env.crash();
const models2 = fake({ respond: echoScript });
env = await open({ dir: env.dir, backend: "jsonl", models: models2 });
await env.root.waitForIdle(ctx);
assert.ok(Date.now() >= untilMs, "waited out the durable deadline");
assert.equal((await env.input(a.id))?.status, "done");
assert.equal(models2.calls, 1);
assert.equal(kinds(await env.entries()), "user system usage assistant");
});
test("crash in tool@started with replay:safe → tool re-invoked; unsafe → synthetic interrupted result", async () => {
for (const replay of ["safe", "unsafe"] as const) {
const gate = new Gate();
const t = tool("x", { replay, gate });
let env = await open({ backend: "jsonl", tools: [t] });
onTestFinished(() => env.close().catch(() => {}));
const a = await env.root.send({ content: "tool:x" }, ctx);
await untilPhase(env, "pi.tool", "started");
await gate.arrivals(1);
await env.crash();
const t2 = tool("x", { replay });
env = await open({ dir: env.dir, backend: "jsonl", tools: [t2] });
await env.root.waitForIdle(ctx); // `a` belongs to the crashed harness; handles do not survive a reopen
assert.equal((await env.input(a.id))?.status, "done");
const result = (await env.entries()).find((e) => e.kind === "pi.tool_result")!;
if (replay === "safe") {
assert.equal(t2.calls, 1);
assert.match(contentOf(result), /x\(x\)/);
} else {
assert.equal(t2.calls, 0);
assert.match(contentOf(result), /interrupted/);
}
assert.equal(kinds(await env.entries()), "user system assistant tool_result assistant");
await env.close();
}
});
test("crash in tool before `started` (blocked in beforeTool) → hook reruns, tool runs once", async () => {
const gate = new Gate();
const t = tool("x");
let hookCalls = 0;
const hooks = {
tool: {
beforeTool: async (_c: unknown, _i: unknown, c: Context) => {
hookCalls++;
await gate.wait(c);
return undefined;
},
},
};
let env = await open({ backend: "jsonl", tools: [t], hooks });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "tool:x" }, ctx);
await untilPhase(env, "pi.tool", undefined);
await gate.arrivals(1);
await env.crash();
gate.open();
const t2 = tool("x");
env = await open({ dir: env.dir, backend: "jsonl", tools: [t2], hooks });
await env.root.waitForIdle(ctx);
assert.equal(hookCalls, 2);
assert.equal(t2.calls, 1);
assert.equal((await env.input(a.id))?.status, "done");
});
test("crash while post_tools is running (blocked in afterTools) → reruns from initial, continuation runs once", async () => {
const gate = new Gate();
let hookCalls = 0;
const hooks = {
postTools: {
afterTools: async (_a: unknown, _r: unknown, _i: unknown, c: Context) => {
hookCalls++;
await gate.wait(c);
},
},
};
let env = await open({ backend: "jsonl", tools: [tool("x")], hooks });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "tool:x" }, ctx);
await gate.arrivals(1); // post_tools is running, no checkpoint
const live = (await env.tasks()).filter((t) => t.status !== "terminal");
assert.deepEqual(
live.map((t) => t.kind),
["pi.post_tools"],
);
await env.crash();
gate.open();
env = await open({ dir: env.dir, backend: "jsonl", tools: [tool("x")], hooks });
// at open: post_tools is live, its tools are retained (referenced by `after`), the old generation is pruned
const atOpen = await env.tasks();
assert.deepEqual(
atOpen
.filter((t) => t.kind !== "pi.generation")
.map((t) => `${t.kind.replace("pi.", "")}:${t.status}`)
.sort(),
["post_tools:running", "tool:terminal"],
);
await env.root.waitForIdle(ctx);
assert.equal(hookCalls, 2);
assert.equal((await env.input(a.id))?.status, "done");
assert.equal(kinds(await env.entries()), "user system assistant tool_result assistant");
const pt = (await env.tasks()).find((t) => t.kind === "pi.post_tools")!;
assert.equal((await env.tasks()).filter((t) => t.kind === "pi.generation" && t.id > pt.id).length, 1); // exactly one continuation (the original generation is retained: no pruning)
});
test("crash in collapse@summarizing → counted as a failed attempt, retried, summary lands", async () => {
const gate = new Gate();
const isSummary = (m: RequestMessage[]) =>
m.some((x) => x.role === "user" && String((x as { content: unknown }).content).startsWith("Summarize"));
let env = await open({
backend: "jsonl",
models: fake({ respond: (m) => (isSummary(m) ? { text: "the summary" } : echoScript(m)), gate }),
root: {
rewindable: { model, keepRecent: 10 },
sticky: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } },
},
});
onTestFinished(() => env.close());
gate.open();
for (const t of ["one", "two", "three"]) await (await env.root.send({ content: t }, ctx)).wait(ctx);
gate.close();
const cid = await env.root.collapse(undefined, ctx);
await untilPhase(env, "pi.collapse", "summarizing");
await gate.arrivals(4);
await env.crash();
env = await open({
dir: env.dir,
backend: "jsonl",
models: fake({ respond: (m) => (isSummary(m) ? { text: "the summary" } : echoScript(m)) }),
});
const c = await untilTerminal(env, cid);
assert.equal(c.outcome?.status, "completed");
const summary = (await env.entries()).find((e) => e.kind === "pi.summary")!;
assert.equal(contentOf(summary), "the summary ");
assert.ok(summary.head! > 0);
});
test("crash in job@spawning / @running → host.status decides; unknown+rerun restarts with the same key", async () => {
const host = fakeHost();
let env = await open({ backend: "jsonl", processHost: host });
onTestFinished(() => env.close());
const jobId = await env.root.commit(
(tx) =>
tx.createTask(
kindsOf.job,
{ command: "x", args: [], cwd: "/", notify: true, rerun: true },
{ conversationId: 1, background: true },
).id,
ctx,
);
const running = await untilPhase(env, "pi.job", "running");
const key = (running.checkpoint as unknown as { key: string }).key;
assert.equal(host.startCalls, 1);
await env.crash();
// Reopen with a host that forgot the process: unknown + rerun → start again, same key.
const host2 = fakeHost();
env = await open({ dir: env.dir, backend: "jsonl", processHost: host2 });
await sleep(50);
assert.equal(host2.startCalls, 1);
assert.ok(host2.procs.has(key));
host2.exit(key, 0);
const done = await untilTerminal(env, jobId);
assert.deepEqual(done.outcome, {
status: "completed",
result: { exitCode: 0, occurrences: 1, stdout: `out ${key}`, stderr: "" },
});
// notice was written (idle conversation → appended immediately)
assert.equal(kinds(await env.entries()), "notice");
});
test("job with rerun:false and unknown after crash → failed/interrupted", async () => {
const host = fakeHost();
let env = await open({ backend: "jsonl", processHost: host });
onTestFinished(() => env.close());
const jobId = await env.root.commit(
(tx) =>
tx.createTask(
kindsOf.job,
{ command: "x", args: [], cwd: "/", notify: false, rerun: false },
{ conversationId: 1, background: true },
).id,
ctx,
);
await untilPhase(env, "pi.job", "running");
await env.crash();
env = await open({ dir: env.dir, backend: "jsonl", processHost: fakeHost() });
assert.equal(
((await untilTerminal(env, jobId)).outcome as unknown as { failure: { reason: string } }).failure.reason,
"interrupted",
);
});
test("reopen with a marked task runs abort, not run", async () => {
const gate = new Gate();
let env = await open({ backend: "jsonl", models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
const t = await untilPhase(env, "pi.generation", "requesting");
await gate.arrivals(1);
// Mark durably, then crash before the fresh abort gets to run.
await env.h.markTask(t.id, ctx);
await env.crash();
const models2 = fake({ respond: echoScript });
env = await open({ dir: env.dir, backend: "jsonl", models: models2 });
assert.equal((await untilTerminal(env, t.id)).outcome?.status, "aborted");
assert.equal(models2.calls, 0); // never re-requested
assert.equal((await env.input(a.id))?.reason, "aborted");
});
test("memory vs jsonl: identical transcripts and outcomes for the same script", async () => {
const results: string[] = [];
for (const backend of ["memory", "jsonl"] as const) {
const t = tool("a");
const env = await open({ backend, tools: [t] });
onTestFinished(() => env.close().catch(() => {}));
for (const c of ["tool:a", "plain", "tool:a"]) await (await env.root.send({ content: c }, ctx)).wait(ctx);
await env.root.waitForIdle(ctx);
results.push(
`${kinds(await env.entries())}|${(await env.tasks()).map((x) => `${x.kind}:${x.outcome?.status}`).join(",")}`,
);
await env.close();
}
assert.equal(results[0], results[1]);
});

View file

@ -0,0 +1,55 @@
import assert from "node:assert/strict";
import { readdirSync } from "node:fs";
import { onTestFinished, test } from "vitest";
import { kinds } from "../../../src/harness/pico3/harness.ts";
import { defineTask } from "../../../src/harness/pico3/types.ts";
import { ctx, open, resultOf } from "./helpers.ts";
test("terminal tasks stay readable (getTask, waitForTask, scans, typed outcome) after many retirements and after reopen; sidecars are unlinked", async () => {
const k = defineTask<{ n: number }, { phase: "x" }, { n: number }, null, null>({
name: "n",
async initial() {
return { next: { phase: "x" } };
},
phases: {
async x(task) {
return { done: () => ({ status: "completed", result: { n: task.input.n } }) };
},
},
async abort() {
return () => null;
},
});
let env = await open({ backend: "jsonl", taskKinds: [k] });
onTestFinished(() => env.close());
const first = await env.root.commit(
(tx) => tx.createTask(k, { n: 1 }, { conversationId: 1, background: true }),
ctx,
);
await env.h.waitForTask(first.id, ctx);
for (let i = 2; i <= 150; i++) {
const ref = await env.root.commit(
(tx) => tx.createTask(k, { n: i }, { conversationId: 1, background: true }),
ctx,
);
await env.h.waitForTask(ref.id, ctx);
}
assert.deepEqual(resultOf(await env.h.getTask(first.id, ctx)), { n: 1 });
assert.equal((await env.tasks()).filter((t) => t.status === "terminal").length, 150);
const plugin = await env.root.commit(
(tx) => tx.createTask(kinds.plugin, { handler: "none", input: null }, { conversationId: 1, background: true }),
ctx,
);
await env.h.waitForTask(plugin.id, ctx);
await env.crash();
assert.deepEqual(
readdirSync(env.dir!).filter((f) => f.startsWith("task-")),
[],
); // every live-only sidecar unlinked after its terminal record
env = await open({ dir: env.dir, backend: "jsonl", taskKinds: [k] });
assert.deepEqual(resultOf(await env.h.getTask(first.id, ctx)), { n: 1 });
assert.equal((await env.h.waitForTask(first.id, ctx)).status, "terminal");
assert.equal((await env.h.getTask(plugin.id, ctx))?.outcome?.status, "failed"); // an outcome not represented by any entry survives
assert.equal((await env.tasks()).filter((t) => t.kind === "n").length, 150);
assert.equal((await env.h.getTask(first.id, ctx))?.checkpoint, undefined); // terminal checkpoints are dropped
});

View file

@ -0,0 +1,233 @@
import assert from "node:assert/strict";
import { createContextKey, withContextValue } from "@earendil-works/chord/context";
import { onTestFinished, test } from "vitest";
import { NestedLineOperation } from "../../../src/harness/pico3/session.ts";
import { type CoreTx, defineTask, Forbidden, type Namespace, type Runtime } from "../../../src/harness/pico3/types.ts";
import { ctx, Gate, open } from "./helpers.ts";
test("line context is propagated and cannot re-enter commit, reads, watches, or waiter registration", async () => {
const env = await open({});
onTestFinished(() => env.close());
const callerKey = createContextKey<string>("pico3.spec.caller");
const callerContext = withContextValue(callerKey, "caller-value", ctx);
let lineContext = ctx;
await env.root.commit((_tx, currentContext) => {
assert.equal(currentContext.value(callerKey), "caller-value");
lineContext = currentContext;
}, callerContext);
await assert.rejects(
env.root.commit(() => undefined, lineContext),
NestedLineOperation,
);
await assert.rejects(env.root.write({ kind: "note" }, lineContext), NestedLineOperation);
await assert.rejects(env.root.watch(lineContext), NestedLineOperation);
await assert.rejects(env.h.root(lineContext), NestedLineOperation);
await assert.rejects(env.h.getTask(999_999, lineContext), NestedLineOperation);
await assert.rejects(env.h.waitForTask(999_999, lineContext), NestedLineOperation);
await assert.rejects(env.root.waitForIdle(lineContext), NestedLineOperation);
});
test("ordinary task scope covers its owned subtree but rejects every foreign direct and off-line read after malicious casts", async () => {
const results = new Map<string, unknown>();
let foreignConversation = 0;
let foreignEntry = 0;
let foreignInput = 0;
let foreignTask = 0;
let child = 0;
let grandchild = 0;
const probe = defineTask<null, { phase: "done" }, null, null, null>({
name: "spec.scope.probe",
async initial(_task, runtime, context) {
child = await runtime.commit((tx) => tx.createConversation({}), context);
grandchild = await runtime.commit(
(tx) => tx.createConversation({ parent: { conversationId: child, at: "start" } }),
context,
);
await runtime.commit(async (tx) => {
await tx.write(child, { kind: "owned.child" });
await tx.write(grandchild, { kind: "owned.grandchild" });
assert.equal((await tx.conversation(child))?.owner, runtime.taskId);
assert.equal((await tx.conversation(grandchild))?.owner, runtime.taskId);
}, context);
await runtime.context(grandchild, undefined, context);
await runtime.rewindable(child, context);
await runtime.sticky(grandchild, context);
const attempts: Record<string, () => Promise<unknown>> = {
"tx.conversation": () => runtime.commit((tx) => tx.conversation(foreignConversation), context),
"tx.entry": () => runtime.commit((tx) => tx.entry(foreignEntry), context),
"tx.entries": () => runtime.commit((tx) => tx.entries([foreignEntry]), context),
"tx.input": () => runtime.commit((tx) => tx.input(foreignInput), context),
"tx.task": () => runtime.commit((tx) => tx.task(foreignTask), context),
"tx.snapshot": () =>
runtime.commit((tx) => tx.snapshot({ doc: "sticky", conversationId: foreignConversation }), context),
"tx.config": () => runtime.commit((tx) => tx.config(foreignConversation).get("profile"), context),
"tx.write": () => runtime.commit((tx) => tx.write(foreignConversation, { kind: "intrusion" }), context),
"tx.createTask": () =>
runtime.commit(
(tx) => tx.createTask(probe, null, { conversationId: foreignConversation, background: true }),
context,
),
"tx.createConversation(parent)": () =>
runtime.commit(
(tx) => tx.createConversation({ parent: { conversationId: foreignConversation, at: "start" } }),
context,
),
"runtime.context": () => runtime.context(foreignConversation, undefined, context),
"runtime.newestEntry": () => runtime.newestEntry(foreignConversation, {}, context),
"runtime.rewindable": () => runtime.rewindable(foreignConversation, context),
"runtime.sticky": () => runtime.sticky(foreignConversation, context),
"runtime.rewindableAsOf": () => runtime.rewindableAsOf(foreignConversation, foreignEntry, context),
"runtime.sendOwned": () => runtime.sendOwned(foreignConversation, { content: "intrusion" }, context),
"runtime.abortConversation": () => runtime.abortConversation(foreignConversation, context),
};
for (const [name, attempt] of Object.entries(attempts)) {
try {
await attempt();
results.set(name, "allowed");
} catch (error) {
results.set(name, error);
}
}
return { next: { phase: "done" } };
},
phases: {
async done() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const foreignKind = defineTask<null, { phase: "done" }, null, null, null>({
name: "spec.scope.foreign-task",
async initial() {
return { next: { phase: "done" } };
},
phases: {
async done() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [probe, foreignKind] });
onTestFinished(() => env.close());
const foreign = await env.h.createConversation({}, ctx);
foreignConversation = foreign.id;
foreignEntry = await foreign.write({ kind: "foreign.entry" }, ctx).then(async (inputId) => {
foreignInput = inputId;
return (await foreign.commit((tx) => tx.input(inputId), ctx))!.entry!;
});
foreignTask = (
await foreign.commit(
(tx) => tx.createTask(foreignKind, null, { conversationId: foreign.id, background: true }),
ctx,
)
).id;
await env.h.waitForTask(foreignTask, ctx);
const ref = await env.root.commit(
(tx) => tx.createTask(probe, null, { conversationId: env.root.id, background: true }),
ctx,
);
await env.h.waitForTask(ref.id, ctx);
assert.ok(child > 0 && grandchild > 0);
assert.equal(results.size, 17);
for (const [name, result] of results) {
assert.ok(result instanceof Forbidden, `${name} unexpectedly produced ${String(result)}`);
}
});
test("a runtime captured by one phase expires when that phase method returns, not only when the whole phase loop ends", async () => {
const phaseGate = new Gate();
let captured: Runtime | undefined;
let state!: Namespace<{ latePhaseWritten: boolean }>;
const kind = defineTask<null, { phase: "later" }, null, null, null>({
name: "spec.phase.capability-lifetime",
async initial(_task, runtime) {
captured = runtime;
return { next: { phase: "later" } };
},
phases: {
async later(_task, _runtime, context) {
await phaseGate.wait(context);
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [kind] });
onTestFinished(() => env.close());
state = env.h.namespace("test.phase-capability", { sticky: { latePhaseWritten: false } });
const ref = await env.root.commit(
(tx) => tx.createTask(kind, null, { conversationId: env.root.id, background: true }),
ctx,
);
await phaseGate.arrivals(1);
let lateError: unknown;
try {
await captured!.commit((tx) => {
tx.plugins(state).latePhaseWritten = true;
}, ctx);
} catch (error) {
lateError = error;
}
phaseGate.open();
await env.h.waitForTask(ref.id, ctx);
assert.ok(lateError instanceof Forbidden, `late phase runtime was accepted: ${String(lateError)}`);
assert.equal((await env.root.sticky(ctx)).plugins["test.phase-capability"], undefined);
});
test("runtime capability checks remain authoritative after structural casts and forged task metadata", async () => {
const observed: unknown[] = [];
const definition = defineTask<null, { phase: "done" }, null, null, null>({
name: "spec.cast.authority",
async initial(_task, runtime, context) {
for (const operation of [
(tx: CoreTx) => tx.appendEntry(runtime.conversationId, { kind: "forged.entry" }),
(tx: CoreTx) => tx.send(runtime.conversationId, { content: "forged input" }),
(tx: CoreTx) => tx.resolveInputs([1], { status: "done", answer: 1 }),
(tx: CoreTx) => tx.markTask(runtime.taskId),
]) {
try {
await runtime.commit((tx) => operation(tx as unknown as CoreTx), context);
observed.push("allowed");
} catch (error) {
observed.push(error);
}
}
return { next: { phase: "done" } };
},
phases: {
async done() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const kind = { ...definition, core: true };
const env = await open({ taskKinds: [kind] });
onTestFinished(() => env.close());
const ref = await env.root.commit(
(tx) => tx.createTask(kind, null, { conversationId: env.root.id, background: true }),
ctx,
);
await env.h.waitForTask(ref.id, ctx);
assert.equal(observed.length, 4);
assert.ok(observed.every((value) => value instanceof Forbidden));
});

View file

@ -0,0 +1,625 @@
import assert from "node:assert/strict";
import { Type } from "typebox";
import { onTestFinished, test } from "vitest";
import { applyEnvelope, kinds } from "../../../src/harness/pico3/harness.ts";
import { JsonlStorage } from "../../../src/harness/pico3/jsonl.ts";
import {
defineTask,
Forbidden,
type JsonObject,
type JsonValue,
type KindConfig,
memoOnce,
type Namespace,
type ToolApi,
type ToolDeclaration,
} from "../../../src/harness/pico3/types.ts";
import { contentOf, ctx, Gate, model, open } from "./helpers.ts";
type MemoToolApi = ToolApi & {
memo<T extends JsonValue>(name: string, candidate: T, context: typeof ctx): Promise<T>;
memo<T extends JsonValue>(name: string, context: typeof ctx): Promise<T | undefined>;
};
const cloneJson = <T>(value: T): T => JSON.parse(JSON.stringify(value)) as T;
type WaitingApi = {
waiting(context: typeof ctx): Promise<void>;
memo<T extends JsonValue>(name: string, candidate: T, context: typeof ctx): Promise<T>;
emit(name: string, data: JsonValue, context: typeof ctx): Promise<void>;
};
test("namespace defaults route to their declared documents, preserve null, and seed lazily", async () => {
const env = await open({});
onTestFinished(() => env.close());
const ns = env.h.namespace<{
plan: { enabled: boolean };
cache: string | null;
global: { count: number };
}>("spec.routing", {
rewindable: { plan: { enabled: false } },
sticky: { cache: null },
session: { global: { count: 0 } },
});
const first = await env.root.commit((tx) => {
const slice = tx.plugins(ns);
assert.equal(slice.cache, null);
slice.plan.enabled = true;
slice.global.count++;
return cloneJson(slice);
}, ctx);
assert.deepEqual(first, { plan: { enabled: true }, cache: null, global: { count: 1 } });
assert.deepEqual((await env.root.rewindable(ctx)).plugins[ns.id], { plan: { enabled: true } });
assert.deepEqual((await env.root.sticky(ctx)).plugins[ns.id], { cache: null });
const sessionSlice = await env.root.commit((tx) => cloneJson(tx.plugins(ns).global), ctx);
assert.deepEqual(sessionSlice, { count: 1 });
});
test("namespace token identity is current authority; unregister is idempotent; re-register retains state and adds defaults", async () => {
const env = await open({});
onTestFinished(() => env.close());
const old = env.h.namespace<{ value: number }>("spec.reload", { rewindable: { value: 1 } });
await env.root.commit((tx) => {
tx.plugins(old).value = 7;
}, ctx);
old.unregister();
old.unregister();
await assert.rejects(
env.root.commit((tx) => tx.plugins(old), ctx),
Forbidden,
);
const current = env.h.namespace<{ value: number; added: string }>("spec.reload", {
rewindable: { value: 100, added: "new-default" },
});
assert.deepEqual(await env.root.commit((tx) => cloneJson(tx.plugins(current)), ctx), {
value: 7,
added: "new-default",
});
old.unregister();
await env.root.commit((tx) => {
tx.plugins(current).added = "still-current";
}, ctx);
assert.equal(await env.root.commit((tx) => tx.plugins(current).added, ctx), "still-current");
});
test("namespace declarations reject ambiguous routing, reserved names, duplicate live registrations, and stale emits", async () => {
const env = await open({});
onTestFinished(() => env.close());
assert.throws(() => env.h.namespace("pi.private", {}), /invalid namespace/);
assert.throws(() => env.h.namespace("bad space", {}), /invalid namespace/);
assert.throws(
() =>
env.h.namespace<{ duplicate: number }>("spec.ambiguous", {
rewindable: { duplicate: 1 },
sticky: { duplicate: 2 },
}),
/more than one document/,
);
const token = env.h.namespace<{ value: number }>("spec.unique", { sticky: { value: 0 } });
assert.throws(() => env.h.namespace("spec.unique", {}), /already registered/);
token.unregister();
await assert.rejects(
env.root.commit((tx) => tx.emit(token, "late", { value: true }), ctx),
Forbidden,
);
});
test("namespace view projection is the only public plugin state and emit produces a namespaced event in the same envelope", async () => {
const env = await open({});
onTestFinished(() => env.close());
const ns = env.h.namespace<{ visible: number; secret: string }>(
"spec.presentation",
{ rewindable: { visible: 0 }, sticky: { secret: "hidden" } },
{ view: (slice) => ({ visible: slice.visible }) },
);
const watch = await env.root.watch(ctx);
const deliveries: Array<{ events?: { type: string; data?: JsonValue }[]; view: object }> = [];
let folded = watch.view;
watch.start((envelope) => {
folded = applyEnvelope(folded, envelope);
deliveries.push({
events: (envelope as unknown as { events?: { type: string; data?: JsonValue }[] }).events,
view: folded,
});
});
await env.root.commit((tx) => {
const slice = tx.plugins(ns);
slice.visible = 2;
slice.secret = "do-not-project";
tx.emit(ns, "changed", { visible: 2 });
}, ctx);
assert.equal(deliveries.length, 1);
const flat = folded as unknown as { plugins: Record<string, JsonValue> };
assert.deepEqual(flat.plugins[ns.id], { visible: 2 });
assert.doesNotMatch(JSON.stringify(flat.plugins), /do-not-project/);
assert.deepEqual(deliveries[0]!.events, [{ type: "plugin.spec.presentation.changed", data: { visible: 2 } }]);
watch.stop();
});
test("memoOnce is first-writer-wins for null and isolates equal names in different slots", () => {
const first: JsonObject = {};
const second: JsonObject = {};
assert.equal(memoOnce(first, "decision", null), null);
assert.equal(memoOnce(first, "decision", "later"), null);
assert.equal(memoOnce(second, "decision", "other-task"), "other-task");
assert.deepEqual(first, { memos: { decision: null } });
assert.deepEqual(second, { memos: { decision: "other-task" } });
});
test("ToolApi.memo serializes concurrent writers, returns the durable winner, and isolates later tool tasks", async () => {
const schema = Type.Object({ v: Type.String() });
const winners: JsonValue[][] = [];
const declaration: ToolDeclaration<typeof schema> = {
name: "memo-tool",
description: "",
parameters: schema,
replay: "safe",
async execute(_args, rawApi, context) {
const api = rawApi as MemoToolApi;
const pair = await Promise.all([api.memo("winner", "first", context), api.memo("winner", "second", context)]);
winners.push(pair);
const stored = await api.memo<JsonValue>("winner", context);
return { content: [{ type: "text", text: JSON.stringify({ pair, stored }) }] };
},
};
const env = await open({ tools: [declaration], root: { rewindable: { model, selectedTools: [declaration.name] } } });
onTestFinished(() => env.close());
await (await env.root.send({ content: "tool:memo-tool" }, ctx)).wait(ctx);
await (await env.root.send({ content: "tool:memo-tool" }, ctx)).wait(ctx);
assert.deepEqual(winners, [
["first", "first"],
["first", "first"],
]);
const toolTasks = (await env.tasks()).filter((task) => task.kind === "pi.tool");
assert.equal(toolTasks.length, 2);
const sticky = await env.root.sticky(ctx);
for (const task of toolTasks) assert.equal(sticky.tasks[task.id], undefined, "terminal slot and memos retired");
});
test("a safe tool memo survives recovery and prevents repeating the coordinated external decision", async () => {
const gate = new Gate();
let asks = 0;
let missingApi = false;
const makeTool = (block: boolean): ToolDeclaration => ({
name: "recover-memo",
description: "",
parameters: Type.Object({ v: Type.String() }),
replay: "safe",
async execute(_args, rawApi, context) {
const api = rawApi as Partial<MemoToolApi>;
if (typeof api.memo !== "function") {
missingApi = true;
asks++;
if (block) await gate.wait(context);
return { content: [{ type: "text", text: "memo unavailable" }] };
}
let decision = await api.memo<string>("decision", context);
if (decision === undefined) {
asks++;
decision = await api.memo("decision", "approved", context);
if (block) await gate.wait(context);
}
return { content: [{ type: "text", text: decision }] };
},
});
let env = await open({ backend: "jsonl", tools: [makeTool(true)] });
onTestFinished(() => env.close());
await env.root.send({ content: "tool:recover-memo" }, ctx);
await gate.arrivals(1);
await env.crash();
env = await open({ dir: env.dir, backend: "jsonl", tools: [makeTool(false)] });
await env.root.waitForIdle(ctx);
assert.equal(missingApi, false);
assert.equal(asks, 1, "recovery reads the first invocation's durable memo instead of asking again");
const result = (await env.entries()).find((entry) => entry.kind === "pi.tool_result")!;
assert.match(contentOf(result), /approved/);
});
test("beforeTool waiting/memo/emit are namespace-bound and waitingOn clears atomically with started", async () => {
const toolGate = new Gate();
const declaration: ToolDeclaration = {
name: "approved-tool",
description: "",
parameters: Type.Object({ v: Type.String() }),
replay: "safe",
async execute(_args, _api, context) {
await toolGate.wait(context);
return { content: [{ type: "text", text: "ran" }] };
},
};
const env = await open({ tools: [declaration] });
onTestFinished(() => env.close());
const ns = env.h.namespace<{ approvals: number }>("spec.approval", { sticky: { approvals: 0 } });
const harnessWithNamespacedHooks = env.h as unknown as {
hooks<T extends JsonObject>(
namespace: Namespace<T>,
kind: typeof kinds.tool,
handlers: { beforeTool(call: unknown, api: WaitingApi, context: typeof ctx): Promise<void> },
): () => void;
};
let releaseApproval!: () => void;
const approval = new Promise<void>((resolve) => {
releaseApproval = resolve;
});
const off = harnessWithNamespacedHooks.hooks(ns, kinds.tool, {
async beforeTool(_call, api, context) {
await api.waiting(context);
await approval;
await api.memo("decision", "allow", context);
await api.emit("approved", { by: "spec" }, context);
},
});
const input = await env.root.send({ content: "tool:approved-tool" }, ctx);
for (let i = 0; i < 100; i++) {
const waiting = (await env.root.sticky(ctx)).turn.tools[0]?.waitingOn;
if (waiting === ns.id) break;
await new Promise((resolve) => setTimeout(resolve, 1));
}
assert.equal((await env.root.sticky(ctx)).turn.tools[0]?.waitingOn, ns.id);
releaseApproval();
await toolGate.arrivals(1);
const slot = (await env.root.sticky(ctx)).turn.tools[0]!;
assert.equal(slot.status, "running");
assert.equal(slot.waitingOn, undefined);
toolGate.open();
await input.wait(ctx);
off();
});
test("a throwing beforeTool handler clears waitingOn in the same synthetic-result envelope", async () => {
let toolCalls = 0;
const declaration: ToolDeclaration = {
name: "blocked-after-wait",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute() {
toolCalls++;
return { content: [{ type: "text", text: "must not run" }] };
},
};
const env = await open({ tools: [declaration] });
onTestFinished(() => env.close());
const ns = env.h.namespace("spec.throwing-approval", {});
const namespaced = env.h as unknown as {
hooks<T extends JsonObject>(
namespace: Namespace<T>,
kind: typeof kinds.tool,
handlers: { beforeTool(call: unknown, api: WaitingApi, context: typeof ctx): Promise<void> },
): () => void;
};
const off = namespaced.hooks(ns, kinds.tool, {
async beforeTool(_call, api, context) {
await api.waiting(context);
throw new Error("approval service failed");
},
});
const watch = await env.root.watch(ctx);
const envelopes: Array<{ ops: unknown[]; events: Array<{ type: string }> }> = [];
watch.start((envelope) =>
envelopes.push(envelope as unknown as { ops: unknown[]; events: Array<{ type: string }> }),
);
await (await env.root.send({ content: "tool:blocked-after-wait" }, ctx)).wait(ctx);
assert.equal(toolCalls, 0);
const result = (await env.entries()).find((entry) => entry.kind === "pi.tool_result")!;
assert.match(contentOf(result), /hook threw: Error: approval service failed/);
const finish = envelopes.find((envelope) => envelope.events.some((event) => event.type === "tool.finished"));
assert.ok(finish);
assert.match(JSON.stringify(finish.ops), /waitingOn/);
assert.equal((await env.root.sticky(ctx)).turn.tools.length, 0);
watch.stop();
off();
});
test("abort unwinds a waiting beforeTool handler and does not leave waitingOn durable", async () => {
const waitingGate = new Gate();
const declaration: ToolDeclaration = {
name: "wait-forever",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute() {
return { content: [] };
},
};
const env = await open({ tools: [declaration] });
onTestFinished(() => env.close());
const ns = env.h.namespace("spec.abort-approval", {});
const namespaced = env.h as unknown as {
hooks<T extends JsonObject>(
namespace: Namespace<T>,
kind: typeof kinds.tool,
handlers: { beforeTool(call: unknown, api: WaitingApi, context: typeof ctx): Promise<void> },
): () => void;
};
namespaced.hooks(ns, kinds.tool, {
async beforeTool(_call, api, context) {
await api.waiting(context);
await waitingGate.wait(context);
},
});
const input = await env.root.send({ content: "tool:wait-forever" }, ctx);
for (let i = 0; i < 100; i++) {
if ((await env.root.sticky(ctx)).turn.tools[0]?.waitingOn === ns.id) break;
await new Promise((resolve) => setTimeout(resolve, 1));
}
assert.equal((await env.root.sticky(ctx)).turn.tools[0]?.waitingOn, ns.id);
await env.root.abort(ctx);
assert.equal((await input.result(ctx))?.reason, "aborted");
assert.deepEqual((await env.root.sticky(ctx)).turn, { tools: [] });
});
test("suspend clears a waiting hook without terminalizing its durable tool task", async () => {
const gate = new Gate();
const declaration: ToolDeclaration = {
name: "suspended-approval",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute() {
return { content: [] };
},
};
const env = await open({ backend: "jsonl", tools: [declaration] });
onTestFinished(() => env.close());
const ns = env.h.namespace("spec.suspended-approval", {});
env.h.hooks(ns, kinds.tool, {
async beforeTool(_call, api, context) {
await api.waiting(context);
await gate.wait(context);
},
});
await env.root.send({ content: "tool:suspended-approval" }, ctx);
for (let attempt = 0; attempt < 100; attempt++) {
if ((await env.root.sticky(ctx)).turn.tools[0]?.waitingOn === ns.id) break;
await new Promise((resolve) => setTimeout(resolve, 1));
}
const task = (await env.tasks()).find((candidate) => candidate.kind === "pi.tool")!;
await env.h.suspend(ctx);
const storage = await JsonlStorage.open(env.dir!, { fsync: false });
try {
const sticky = await storage.doc({ doc: "sticky", conversationId: 1 }, ctx);
const tools = (sticky as { turn: { tools: Array<{ waitingOn?: string }> } }).turn.tools;
assert.equal(tools[0]?.waitingOn, undefined);
assert.equal((await storage.task(task.id, ctx))?.status, "running");
} finally {
await storage.close(ctx);
}
});
test("equal hook memo names are isolated by namespace on one hosting tool task", async () => {
const observed: Record<string, JsonValue | undefined> = {};
const declaration: ToolDeclaration = {
name: "two-approvals",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute() {
return { content: [{ type: "text", text: "ran" }] };
},
};
const env = await open({ tools: [declaration] });
onTestFinished(() => env.close());
const namespaced = env.h as unknown as {
hooks<T extends JsonObject>(
namespace: Namespace<T>,
kind: typeof kinds.tool,
handlers: { beforeTool(call: unknown, api: WaitingApi, context: typeof ctx): Promise<void> },
): () => void;
};
for (const id of ["spec.approver-a", "spec.approver-b"] as const) {
const ns = env.h.namespace(id, {});
namespaced.hooks(ns, kinds.tool, {
async beforeTool(_call, api, context) {
await api.memo("decision", id, context);
observed[id] = await (api as unknown as MemoToolApi).memo<JsonValue>("decision", context);
},
});
}
await (await env.root.send({ content: "tool:two-approvals" }, ctx)).wait(ctx);
assert.deepEqual(observed, {
"spec.approver-a": "spec.approver-a",
"spec.approver-b": "spec.approver-b",
});
});
test("custom task describe cannot project private slot memos", async () => {
const gate = new Gate();
type Slot = { progress: number; memos?: { secret: string } };
const kind = defineTask<null, { phase: "never" }, null, null, null, object, KindConfig, Slot>({
name: "spec.private-memo-view",
slot: () => ({ progress: 0 }),
describe: (task) => task.slot ?? null,
async initial(task, runtime, context) {
await runtime.commit((tx) => {
const slot = tx.slot({ id: task.id, kind });
slot.progress = 1;
slot.memos = { secret: "never-project" };
}, context);
await gate.wait(context);
return { done: () => ({ status: "completed", result: null }) };
},
phases: {
async never() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [kind] });
onTestFinished(() => env.close());
const ref = await env.root.commit((tx) => tx.createTask(kind, null, { conversationId: 1, background: true }), ctx);
await gate.arrivals(1);
const watch = await env.root.watch(ctx);
assert.deepEqual(watch.view.tasks[ref.id]?.status, { progress: 1 });
assert.doesNotMatch(JSON.stringify(watch.view), /never-project/);
watch.stop();
gate.open();
await env.h.waitForTask(ref.id, ctx);
});
test("runtime task-kind registration seeds defaults into already-loaded documents before the task reads them", async () => {
let observed: unknown;
const env = await open({});
onTestFinished(() => env.close());
await env.root.rewindable(ctx);
const kind = defineTask<null, { phase: "done" }, null, null, null, object, { rewindable: { enabled: boolean } }>({
name: "spec.runtime-default",
config: { rewindable: { enabled: false } },
async initial(_task, runtime, context) {
observed = await runtime.commit(
(tx, current) => ({
facade: tx.config(current.conversationId).get("enabled"),
document: tx.snapshot({ doc: "rewindable", conversationId: current.conversationId }).enabled,
}),
context,
);
return { next: { phase: "done" } };
},
phases: {
async done() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
env.h.registerTaskKind(kind);
const ref = await env.root.commit(
(tx) => tx.createTask(kind, null, { conversationId: env.root.id, background: true }),
ctx,
);
await env.h.waitForTask(ref.id, ctx);
assert.deepEqual(observed, { facade: false, document: false });
});
test("an already-running kind keeps its captured config authority after unregister", async () => {
const gate = new Gate();
const kind = defineTask<null, { phase: "never" }, string, null, null, object, { sticky: { leaseValue: string } }>({
name: "spec.active-reload",
config: { sticky: { leaseValue: "old-default" } },
async initial(task, runtime, context) {
await gate.wait(context);
const value = await runtime.commit((tx) => tx.config(task.conversationId).get("leaseValue"), context);
return { done: () => ({ status: "completed", result: String(value) }) };
},
phases: {
async never() {
return { done: () => ({ status: "completed", result: "never" }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({});
onTestFinished(() => env.close());
const unregister = env.h.registerTaskKind(kind);
const ref = await env.root.commit((tx) => tx.createTask(kind, null, { conversationId: 1, background: true }), ctx);
await gate.arrivals(1);
unregister();
gate.open();
assert.deepEqual((await env.h.waitForTask(ref.id, ctx)).outcome, {
status: "completed",
result: "old-default",
});
});
test("an unregistered kind leaves dependency-blocked tasks pending until a replacement is registered", async () => {
const gate = new Gate();
const blocker = defineTask<null, { phase: "never" }, null, null, null>({
name: "spec.reload-blocker",
async initial(_task, _runtime, context) {
await gate.wait(context);
return { done: () => ({ status: "completed", result: null }) };
},
phases: {
async never() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const makeKind = (result: string) =>
defineTask<null, { phase: "never" }, string, null, null>({
name: "spec.pending-reload",
async initial() {
return { done: () => ({ status: "completed", result }) };
},
phases: {
async never() {
return { done: () => ({ status: "completed", result }) };
},
},
async abort() {
return () => null;
},
});
const original = makeKind("original");
const env = await open({ taskKinds: [blocker] });
onTestFinished(() => env.close());
const removeOriginal = env.h.registerTaskKind(original);
const refs = await env.root.commit((tx) => {
const dependency = tx.createTask(blocker, null, { conversationId: 1, background: true });
const target = tx.createTask(original, null, {
conversationId: 1,
background: true,
after: [dependency.id],
});
return { dependency, target };
}, ctx);
await gate.arrivals(1);
removeOriginal();
gate.open();
await env.h.waitForTask(refs.dependency.id, ctx);
await new Promise((resolve) => setTimeout(resolve, 5));
assert.equal((await env.h.getTask(refs.target.id, ctx))?.status, "pending");
const replacement = makeKind("replacement");
env.h.registerTaskKind(replacement);
assert.deepEqual((await env.h.waitForTask(refs.target.id, ctx)).outcome, {
status: "completed",
result: "replacement",
});
});
test("task kind unregister/re-register makes old tokens stale without letting an old unsubscribe remove the replacement", async () => {
const calls: string[] = [];
const make = (label: string) =>
defineTask<null, { phase: "done" }, string, null, null>({
name: "spec.reload-kind",
async initial() {
calls.push(label);
return { next: { phase: "done" } };
},
phases: {
async done() {
return { done: () => ({ status: "completed", result: label }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({});
onTestFinished(() => env.close());
const old = make("old");
const unregisterOld = env.h.registerTaskKind(old);
unregisterOld();
unregisterOld();
const replacement = make("replacement");
env.h.registerTaskKind(replacement);
unregisterOld();
await assert.rejects(
env.root.commit((tx) => tx.createTask(old, null, { conversationId: 1 }), ctx),
Forbidden,
);
const ref = await env.root.commit(
(tx) => tx.createTask(replacement, null, { conversationId: 1, background: true }),
ctx,
);
const terminal = await env.h.waitForTask(ref.id, ctx);
assert.deepEqual(terminal.outcome, { status: "completed", result: "replacement" });
assert.deepEqual(calls, ["replacement"]);
});

View file

@ -0,0 +1,340 @@
import assert from "node:assert/strict";
import { onTestFinished, test, vi } from "vitest";
import { kinds } from "../../../src/harness/pico3/harness.ts";
import { CollapseInProgress, defineTask, type ProcessHost } from "../../../src/harness/pico3/types.ts";
import { contentOf, ctx, echoScript, fake, Gate, model, open, tool, untilPhase } from "./helpers.ts";
test("terminal generation failure drains passive writes and settles every queued trigger through successor groups", async () => {
const gate = new Gate();
const models = fake({ respond: () => ({ error: "provider down" }), gate });
const env = await open({
models,
root: {
rewindable: { model },
sticky: {
retry: { enabled: false, maxRetries: 0, baseDelayMs: 1 },
followUpMode: "all",
steeringMode: "all",
},
},
});
onTestFinished(() => env.close());
const first = await env.root.send({ content: "first" }, ctx);
await gate.arrivals(1);
const followA = await env.root.send({ content: "follow-a" }, ctx);
const steer = await env.root.send({ content: "steer", whenBusy: "steer" }, ctx);
const followB = await env.root.send({ content: "follow-b" }, ctx);
const passive = await env.root.write({ kind: "passive.note", data: { durable: true } }, ctx);
await env.root.config.reset(["model"], ctx);
gate.open();
const settled = await Promise.all([first.wait(ctx), followA.wait(ctx), steer.wait(ctx), followB.wait(ctx)]);
assert.deepEqual(
settled.map((input) => input.status),
["unanswered", "unanswered", "unanswered", "unanswered"],
);
assert.deepEqual(
settled.map((input) => input.reason),
["failed", "failed", "failed", "failed"],
);
assert.equal((await env.input(passive))?.status, "done");
assert.deepEqual((await env.root.sticky(ctx)).inbox, []);
const generations = (await env.tasks()).filter((task) => task.kind === "pi.generation");
assert.equal(generations.length, 2, "all three queued triggers form one successor input group");
assert.deepEqual(
generations.map((task) =>
task.outcome?.status === "failed" ? (task.outcome.failure as { reason: string }).reason : task.outcome?.status,
),
["provider", "no_model"],
);
assert.equal(models.calls, 1);
});
test("display-only provider errors remain visible in history but never enter the next provider request", async () => {
let call = 0;
const models = fake({ respond: (messages) => (call++ === 0 ? { error: "fatal" } : echoScript(messages)) });
const env = await open({
models,
root: {
rewindable: { model },
sticky: { retry: { enabled: false, maxRetries: 0, baseDelayMs: 1 } },
},
});
onTestFinished(() => env.close());
const failed = await (await env.root.send({ content: "first" }, ctx)).wait(ctx);
assert.equal(failed.reason, "failed");
const display = (await env.entries()).find(
(entry) => entry.kind === "pi.assistant" && (entry.data as { reason?: string } | undefined)?.reason === "error",
)!;
assert.equal(display.model, undefined);
assert.ok((display.data as { display?: unknown }).display !== undefined);
const second = await (await env.root.send({ content: "second" }, ctx)).wait(ctx);
assert.equal(second.status, "done");
assert.equal(models.requests.length, 2);
assert.ok(
!models.requests[1]!.some(
(message) =>
message.role === "assistant" && (message.stopReason === "error" || message.stopReason === "aborted"),
),
);
assert.doesNotMatch(JSON.stringify(models.requests[1]), /fatal/);
});
test("generation singleton serializes concurrent admissions and permits a new generation after terminal retention", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const first = await env.root.send({ content: "one" }, ctx);
await gate.arrivals(1);
const [second, third] = await Promise.all([
env.root.send({ content: "two" }, ctx),
env.root.send({ content: "three" }, ctx),
]);
const during = (await env.tasks()).filter((task) => task.kind === "pi.generation" && task.status !== "terminal");
assert.equal(during.length, 1);
assert.deepEqual((during[0]!.input as { inputs: number[] }).inputs, [first.id]);
gate.open();
await Promise.all([first.wait(ctx), second.wait(ctx), third.wait(ctx)]);
await env.root.waitForIdle(ctx);
const retained = (await env.tasks()).filter((task) => task.kind === "pi.generation");
assert.equal(retained.length, 3);
assert.equal(
retained.every((task) => task.status === "terminal"),
true,
);
const fourth = await env.root.send({ content: "four" }, ctx);
assert.equal((await fourth.wait(ctx)).status, "done");
});
test("collapse singleton rejects overlap before persistence and can be recreated after the retained terminal task", async () => {
const collapseGate = new Gate();
const summary = (messages: Parameters<typeof echoScript>[0]) =>
messages.some(
(message) =>
message.role === "user" &&
String((message as { content?: unknown }).content).includes("Respond with the summary only"),
)
? { text: "summary" }
: echoScript(messages);
const env = await open({
models: fake({ respond: summary }),
root: { rewindable: { model, keepRecent: 1 } },
hooks: {
collapse: {
beforeCollapse: async (_reason, _through, _entries, _info, context) => {
await collapseGate.wait(context);
},
},
},
});
onTestFinished(() => env.close());
for (const prompt of ["one", "two", "three"]) await (await env.root.send({ content: prompt }, ctx)).wait(ctx);
const first = await env.root.collapse(undefined, ctx);
await collapseGate.arrivals(1);
await assert.rejects(env.root.collapse(undefined, ctx), CollapseInProgress);
collapseGate.open();
const retained = await env.h.waitForTask(first, ctx);
assert.equal(retained.status, "terminal");
assert.equal((await env.h.getTask(first, ctx))?.id, first);
});
test("job spawn failure terminalizes, retains its declared failure, and appends an idle notification without poisoning inbox", async () => {
const host: ProcessHost = {
async start() {
throw new Error("spawn denied");
},
async status() {
return { status: "unknown" };
},
async kill() {},
};
const env = await open({ processHost: host });
onTestFinished(() => env.close());
const ref = await env.root.commit(
(tx) =>
tx.createTask(
kinds.job,
{ command: "x", args: [], cwd: "/", notify: true, rerun: false },
{ conversationId: 1, background: true },
),
ctx,
);
const terminal = await env.h.waitForTask(ref.id, ctx);
assert.deepEqual(terminal.outcome, {
status: "failed",
failure: { reason: "spawn", detail: "Error: spawn denied" },
});
assert.deepEqual((await env.root.sticky(ctx)).inbox, []);
assert.match(contentOf((await env.entries()).find((entry) => entry.kind === "pi.notice")), /failed to start/);
});
test("job host status failure reconciles to interrupted without rerunning the process", async () => {
let starts = 0;
const host: ProcessHost = {
async start() {
starts++;
},
async status() {
throw new Error("host unavailable");
},
async kill() {},
};
const env = await open({ processHost: host });
onTestFinished(() => env.close());
const ref = await env.root.commit(
(tx) =>
tx.createTask(
kinds.job,
{ command: "x", args: [], cwd: "/", notify: false, rerun: true },
{ conversationId: 1, background: true },
),
ctx,
);
const terminal = await env.h.waitForTask(ref.id, ctx);
assert.equal(starts, 1);
assert.deepEqual(terminal.outcome, {
status: "failed",
failure: { reason: "interrupted", detail: "host status failed: Error: host unavailable" },
});
});
test("job abort sends TERM, waits the fixed five-second grace, then sends KILL", async () => {
vi.useFakeTimers();
vi.setSystemTime(10_000);
const kills: Array<{ signal: "SIGTERM" | "SIGKILL"; at: number }> = [];
let resolveStarted!: () => void;
const started = new Promise<void>((resolve) => {
resolveStarted = resolve;
});
let resolveTerm!: () => void;
const term = new Promise<void>((resolve) => {
resolveTerm = resolve;
});
const host: ProcessHost = {
async start() {
resolveStarted();
},
async status() {
return { status: "running", stdout: "", stderr: "", droppedStdout: 0, droppedStderr: 0 };
},
async kill(_key, signal) {
kills.push({ signal, at: Date.now() });
if (signal === "SIGTERM") resolveTerm();
},
};
const env = await open({ processHost: host });
onTestFinished(async () => {
vi.useRealTimers();
await env.close();
});
const ref = await env.root.commit(
(tx) =>
tx.createTask(
kinds.job,
{ command: "x", args: [], cwd: "/", notify: false, rerun: false },
{ conversationId: 1, background: true },
),
ctx,
);
await started;
await Promise.resolve();
await env.h.abortTask(ref.id, ctx);
await term;
assert.deepEqual(kills, [{ signal: "SIGTERM", at: 10_000 }]);
await vi.advanceTimersByTimeAsync(4_999);
assert.equal(kills.length, 1);
await vi.advanceTimersByTimeAsync(1);
const terminal = await env.h.waitForTask(ref.id, ctx);
assert.deepEqual(kills, [
{ signal: "SIGTERM", at: 10_000 },
{ signal: "SIGKILL", at: 15_000 },
]);
assert.deepEqual(terminal.outcome, { status: "aborted", result: { killed: true } });
});
test("suspend unwinds active invocations without terminalizing them and reopen recovers durable work", async () => {
const gate = new Gate();
let env = await open({ backend: "jsonl", tools: [tool("recoverable", { replay: "safe", gate })] });
onTestFinished(() => env.close());
const input = await env.root.send({ content: "tool:recoverable" }, ctx);
const running = await untilPhase(env, "pi.tool", "started");
await gate.arrivals(1);
const lifecycle = env.h as unknown as { suspend(context: typeof ctx): Promise<void> };
await lifecycle.suspend(ctx);
const replacement = tool("recoverable", { replay: "safe" });
env = await open({ dir: env.dir, backend: "jsonl", tools: [replacement] });
await env.root.waitForIdle(ctx);
assert.equal(replacement.calls, 1);
assert.equal((await env.h.getTask(running.id, ctx))?.outcome?.status, "completed");
assert.equal((await env.input(input.id))?.status, "done");
});
test("hold provides a quiescent reload cut: commits continue, dispatch waits, and re-register runs pending records with new code", async () => {
const ran: string[] = [];
const makeKind = (label: string) =>
defineTask<null, { phase: "done" }, string, null, null>({
name: "spec.held-kind",
async initial() {
ran.push(label);
return { next: { phase: "done" } };
},
phases: {
async done() {
return { done: () => ({ status: "completed", result: label }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({});
onTestFinished(() => env.close());
const lifecycle = env.h as unknown as {
quiescent(): boolean;
hold(): () => void;
};
assert.equal(lifecycle.quiescent(), true);
const release = lifecycle.hold();
const old = makeKind("old");
const unregister = env.h.registerTaskKind(old);
const ref = await env.root.commit((tx) => tx.createTask(old, null, { conversationId: 1, background: true }), ctx);
await env.root.write({ kind: "commit.while.held" }, ctx);
assert.equal((await env.h.getTask(ref.id, ctx))?.status, "pending");
assert.deepEqual(ran, []);
unregister();
const replacement = makeKind("replacement");
env.h.registerTaskKind(replacement);
release();
const terminal = await env.h.waitForTask(ref.id, ctx);
assert.deepEqual(terminal.outcome, { status: "completed", result: "replacement" });
assert.deepEqual(ran, ["replacement"]);
assert.equal(lifecycle.quiescent(), true);
});
test("quiescent is false for sleeping jobs and waiting hooks, not merely for actively executing JavaScript", async () => {
const env = await open({
processHost: {
async start() {},
async status() {
return { status: "running", stdout: "", stderr: "", droppedStdout: 0, droppedStderr: 0 };
},
async kill() {},
},
});
onTestFinished(() => env.close());
const lifecycle = env.h as unknown as { quiescent(): boolean };
const ref = await env.root.commit(
(tx) =>
tx.createTask(
kinds.job,
{ command: "x", args: [], cwd: "/", notify: false, rerun: false },
{ conversationId: 1, background: true },
),
ctx,
);
await untilPhase(env, "pi.job", "running");
assert.equal(lifecycle.quiescent(), false);
await env.h.abortTask(ref.id, ctx);
});

View file

@ -0,0 +1,509 @@
import assert from "node:assert/strict";
import { readFileSync, writeFileSync } from "node:fs";
import { join } from "node:path";
import { onTestFinished, test } from "vitest";
import { applyEnvelope, Harness } from "../../../src/harness/pico3/harness.ts";
import { JsonlStorage } from "../../../src/harness/pico3/jsonl.ts";
import { MemoryStorage } from "../../../src/harness/pico3/memory.ts";
import { defineTask, Faulted, type Storage, type Write } from "../../../src/harness/pico3/types.ts";
import { ctx, echoScript, fake, open } from "./helpers.ts";
test("MemoryStorage validates a complete batch before publishing tables, documents, sequence, or id high-water", async () => {
const storage = new MemoryStorage();
const invalid: Write[] = [
{ type: "conversation", conversation: { id: 40 } },
{ type: "doc", ref: { doc: "session" }, ops: [["r", { plugins: { leaked: { value: true } } }]] },
{ type: "entry", entry: { id: 40, conversationId: 40, kind: "duplicate-global-id" } },
];
await assert.rejects(storage.commit(invalid, ctx), /created twice/);
assert.equal(await storage.conversation(40, ctx), undefined);
assert.equal((await storage.entries([40], ctx)).size, 0);
assert.deepEqual(await storage.doc({ doc: "session" }, ctx), { plugins: {} });
await storage.commit([{ type: "conversation", conversation: { id: 1 } }], ctx);
assert.equal(storage.mintId(), 2, "the rejected explicit id 40 did not move committed high-water");
assert.equal(await storage.conversation(1, ctx).then((value) => value?.id), 1);
await storage.close(ctx);
});
test("storage reads are isolated snapshots for every mutable durable family", async () => {
const storage = new MemoryStorage();
await storage.commit(
[
{ type: "conversation", conversation: { id: 1, sections: [{ key: "x", value: { nested: 1 } }] } },
{
type: "entry",
entry: { id: 2, conversationId: 1, kind: "note", data: { nested: { value: 1 } } },
},
{
type: "task",
task: {
id: 3,
conversationId: 1,
kind: "task",
input: { nested: 1 },
status: "pending",
after: [],
owns: [],
},
},
{ type: "input", input: { id: 4, conversationId: 1, status: "queued", requestId: "r" } },
{
type: "doc",
ref: { doc: "rewindable", conversationId: 1 },
ops: [["r", { plugins: { p: { nested: { value: 1 } } } }]],
},
],
ctx,
);
const conversation = await storage.conversation(1, ctx);
const entry = (await storage.entries([2], ctx)).get(2)!;
const task = await storage.task(3, ctx);
const input = await storage.inputByRequest(1, "r", ctx);
const document = await storage.doc({ doc: "rewindable", conversationId: 1 }, ctx);
(conversation!.sections![0] as { value?: unknown }).value = "mutated";
(entry.data!.nested as { value: number }).value = 9;
(task!.input as { nested: number }).nested = 9;
(input as { status: string }).status = "done";
(document!.plugins as { p: { nested: { value: number } } }).p.nested.value = 9;
assert.deepEqual((await storage.conversation(1, ctx))?.sections, [{ key: "x", value: { nested: 1 } }]);
assert.deepEqual((await storage.entries([2], ctx)).get(2)?.data, { nested: { value: 1 } });
assert.deepEqual((await storage.task(3, ctx))?.input, { nested: 1 });
assert.equal((await storage.inputByRequest(1, "r", ctx))?.status, "queued");
assert.deepEqual(await storage.doc({ doc: "rewindable", conversationId: 1 }, ctx), {
plugins: { p: { nested: { value: 1 } } },
});
await storage.close(ctx);
});
test("a rejected persistence cut leaves mutated cached documents unpublished and faults the session", async () => {
class RejectingStorage extends MemoryStorage {
reject = false;
override commit(writes: readonly Write[], context: Parameters<Storage["commit"]>[1]) {
if (this.reject) return Promise.reject(new Error("publication cut"));
return super.commit(writes, context);
}
}
const storage = new RejectingStorage();
const harness = await Harness.open(storage, { models: fake({ respond: echoScript }) }, ctx);
onTestFinished(() => harness.close(ctx).catch(() => {}));
const state = harness.namespace<{ cut: { shouldNotPersist: boolean } }>("test.rejected-cut", {
rewindable: { cut: { shouldNotPersist: false } },
});
harness.resume();
const root = await harness.root(ctx);
storage.reject = true;
await assert.rejects(
root.commit((tx) => {
tx.plugins(state).cut = { shouldNotPersist: true };
}, ctx),
Faulted,
);
const stored = await storage.doc({ doc: "rewindable", conversationId: root.id }, ctx);
assert.equal((stored?.plugins as { "test.rejected-cut"?: unknown })["test.rejected-cut"], undefined);
await assert.rejects(root.write({ kind: "later" }, ctx), Faulted);
});
test("JSONL recovery discards a task sidecar tail when append fails before the sticky sidecar and publication marker", async () => {
const never = defineTask<null, { phase: "never" }, null, null, null>({
name: "never",
async initial() {
return { next: { phase: "never" } };
},
phases: {
async never() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ backend: "jsonl", taskKinds: [never] });
onTestFinished(() => env.close().catch(() => {}));
env.h.hold();
const ref = await env.root.commit(
(tx) => tx.createTask(never, null, { conversationId: env.root.id, background: true }),
ctx,
);
await env.crash();
let storage = await JsonlStorage.open(env.dir!, { fsync: false });
const internals = storage as unknown as { append(fd: number, record: object): void };
const append = internals.append.bind(storage);
let appends = 0;
internals.append = (fd, record) => {
if (++appends === 2) throw new Error("injected sidecar append failure");
append(fd, record);
};
await assert.rejects(
storage.commit(
[
{ type: "task.patch", patch: { id: ref.id, status: "running", checkpoint: { phase: "started" } } },
{
type: "doc",
ref: { doc: "sticky", conversationId: env.root.id },
ops: [["s", ["followUpMode"], "all"]],
},
],
ctx,
),
/injected sidecar append failure/,
);
await storage.close(ctx);
storage = await JsonlStorage.open(env.dir!, { fsync: false });
assert.equal((await storage.task(ref.id, ctx))?.status, "pending");
assert.equal((await storage.task(ref.id, ctx))?.checkpoint, undefined);
assert.equal(
(await storage.doc({ doc: "sticky", conversationId: env.root.id }, ctx))?.followUpMode,
"one-at-a-time",
);
await storage.close(ctx);
});
test("JSONL replay rejects a published marker whose expected sidecar record is missing", async () => {
const env = await open({ backend: "jsonl" });
onTestFinished(() => env.close().catch(() => {}));
await env.root.config.set({ followUpMode: "all" } as never, ctx);
await env.crash();
const sidecar = join(env.dir!, "sticky-1.jsonl");
const records = readFileSync(sidecar, "utf8").split("\n").filter(Boolean);
assert.ok(records.length >= 2);
writeFileSync(sidecar, `${records.slice(0, -1).join("\n")}\n`);
let reopened: JsonlStorage | undefined;
let replayError: unknown;
try {
reopened = await JsonlStorage.open(env.dir!, { fsync: false });
} catch (error) {
replayError = error;
}
await reopened?.close(ctx);
assert.ok(replayError instanceof Error, "the main publication marker must require every named sidecar record");
});
test("unconfirmed JSONL tails are ignored and cannot advance committed id high-water", async () => {
const env = await open({ backend: "jsonl" });
onTestFinished(() => env.close().catch(() => {}));
await env.root.write({ kind: "confirmed" }, ctx);
await env.crash();
const mainRecords = readFileSync(join(env.dir!, "main.jsonl"), "utf8").split("\n").filter(Boolean);
const lastMain = JSON.parse(mainRecords.at(-1)!) as { seq: number; maxId: number };
const sidecar = join(env.dir!, "sticky-999.jsonl");
writeFileSync(
sidecar,
`${JSON.stringify({
seq: lastMain.seq + 1,
maxId: 999_999,
writes: [
{
type: "doc",
ref: { doc: "sticky", conversationId: 999 },
ops: [["r", { inbox: [], turn: { tools: [] }, tasks: {}, plugins: {} }]],
},
],
})}\n`,
);
const storage = await JsonlStorage.open(env.dir!, { fsync: false });
assert.ok(storage.mintId() < 999_999);
assert.equal(await storage.doc({ doc: "sticky", conversationId: 999 }, ctx), undefined);
await storage.close(ctx);
});
test("fork state is commit-granular: an entry observes the final rewindable state of its atomic commit", async () => {
const env = await open({});
onTestFinished(() => env.close());
const state = env.h.namespace<{ version: string }>("test.fork-history", {
rewindable: { version: "" },
});
const committed = await env.root.commit(async (tx) => {
const input = await tx.write(env.root.id, { kind: "anchor" });
tx.plugins(state).version = "same-commit-final";
return input;
}, ctx);
const record = await env.input(committed);
const fork = await env.root.fork(record!.entry!, {}, ctx);
assert.equal((await fork.rewindable(ctx)).plugins["test.fork-history"]?.version, "same-commit-final");
});
test("active transcript capture preserves inherited head targets, inner heads, display-only entries, and edits verbatim", async () => {
const storage = new MemoryStorage();
const emptyRewindable = {
thinkingLevel: "off",
selectedTools: [],
profile: "default",
threshold: 0,
keepRecent: 20_000,
plugins: {},
};
const emptySticky = {
retry: { enabled: false, maxRetries: 0, baseDelayMs: 1 },
steeringMode: "all",
followUpMode: "all",
inbox: [],
turn: { tools: [] },
tasks: {},
plugins: {},
};
await storage.commit(
[
{ type: "conversation", conversation: { id: 1 } },
{ type: "doc", ref: { doc: "rewindable", conversationId: 1 }, ops: [["r", emptyRewindable]] },
{ type: "doc", ref: { doc: "sticky", conversationId: 1 }, ops: [["r", emptySticky]] },
{
type: "entry",
entry: {
id: 2,
conversationId: 1,
kind: "pi.user",
model: [{ role: "user", content: "base", timestamp: 1 }],
},
},
{ type: "entry", entry: { id: 3, conversationId: 1, kind: "pi.summary", head: 2 } },
{
type: "entry",
entry: { id: 4, conversationId: 1, kind: "pi.assistant", data: { reason: "aborted" } },
},
{
type: "entry",
entry: { id: 5, conversationId: 1, kind: "plugin.note", edits: [{ target: 2, action: "omit" }] },
},
{ type: "conversation", conversation: { id: 10, parent: { conversationId: 1, at: 5 } } },
{ type: "doc", ref: { doc: "rewindable", conversationId: 10 }, ops: [["r", emptyRewindable]] },
{ type: "doc", ref: { doc: "sticky", conversationId: 10 }, ops: [["r", emptySticky]] },
{ type: "entry", entry: { id: 11, conversationId: 10, kind: "pi.handoff", head: 2 } },
],
ctx,
);
const harness = await Harness.open(storage, { models: fake({ respond: echoScript }) }, ctx);
onTestFinished(() => harness.close(ctx));
harness.resume();
const child = (await harness.conversation(10, ctx))!;
const watch = await child.watch(ctx);
let folded = watch.view;
watch.start((envelope) => {
folded = applyEnvelope(folded, envelope);
});
assert.deepEqual(
watch.view.entries.map((entry) => entry.id),
[2, 3, 4, 5, 11],
);
assert.deepEqual(watch.view.entries.find((entry) => entry.id === 5)?.edits, [{ target: 2, action: "omit" }]);
assert.equal(watch.view.entries.find((entry) => entry.id === 4)?.model, undefined);
const projected = await child.context(ctx);
assert.equal(projected.head?.id, 11);
assert.ok(
!projected.entries.some((entry) => entry.id === 3),
"model context drops inner heads while the view retains them",
);
await child.reset(undefined, ctx);
const fresh = await child.watch(ctx);
assert.deepEqual(folded.entries, fresh.view.entries);
assert.equal(fresh.view.entries.length, 1);
watch.stop();
fresh.stop();
});
test("active transcript capture has no fixed page cap and crosses the head boundary on an older page", async () => {
const storage = new MemoryStorage();
const rewindable = {
thinkingLevel: "off",
selectedTools: [],
profile: "default",
threshold: 0,
keepRecent: 20_000,
plugins: {},
};
const sticky = {
retry: { enabled: false, maxRetries: 0, baseDelayMs: 1 },
steeringMode: "all",
followUpMode: "all",
inbox: [],
turn: { tools: [] },
tasks: {},
plugins: {},
};
const writes: Write[] = [
{ type: "conversation", conversation: { id: 1 } },
{ type: "doc", ref: { doc: "rewindable", conversationId: 1 }, ops: [["r", rewindable]] },
{ type: "doc", ref: { doc: "sticky", conversationId: 1 }, ops: [["r", sticky]] },
];
for (let id = 2; id <= 602; id++) writes.push({ type: "entry", entry: { id, conversationId: 1, kind: "history" } });
writes.push({ type: "entry", entry: { id: 603, conversationId: 1, kind: "pi.summary", head: 100 } });
await storage.commit(writes, ctx);
const harness = await Harness.open(storage, { models: fake({ respond: echoScript }) }, ctx);
onTestFinished(() => harness.close(ctx));
harness.resume();
const watch = await (await harness.root(ctx)).watch(ctx);
assert.equal(watch.view.entries.length, 504);
assert.equal(watch.view.entries[0]?.id, 100);
assert.equal(watch.view.entries.at(-1)?.id, 603);
watch.stop();
});
test("newest in-range edit wins while the rendering transcript retains every edit verbatim", async () => {
const storage = new MemoryStorage();
const rewindable = {
thinkingLevel: "off",
selectedTools: [],
profile: "default",
threshold: 0,
keepRecent: 20_000,
plugins: {},
};
const sticky = {
retry: { enabled: false, maxRetries: 0, baseDelayMs: 1 },
steeringMode: "all",
followUpMode: "all",
inbox: [],
turn: { tools: [] },
tasks: {},
plugins: {},
};
await storage.commit(
[
{ type: "conversation", conversation: { id: 1 } },
{ type: "doc", ref: { doc: "rewindable", conversationId: 1 }, ops: [["r", rewindable]] },
{ type: "doc", ref: { doc: "sticky", conversationId: 1 }, ops: [["r", sticky]] },
{
type: "entry",
entry: {
id: 2,
conversationId: 1,
kind: "pi.user",
model: [{ role: "user", content: "original", timestamp: 1 }],
},
},
{
type: "entry",
entry: {
id: 3,
conversationId: 1,
kind: "edit.one",
edits: [
{ target: 2, action: "replace", messages: [{ role: "user", content: "older edit", timestamp: 2 }] },
],
},
},
{
type: "entry",
entry: {
id: 4,
conversationId: 1,
kind: "edit.two",
edits: [
{ target: 2, action: "replace", messages: [{ role: "user", content: "newest edit", timestamp: 3 }] },
],
},
},
],
ctx,
);
const harness = await Harness.open(storage, { models: fake({ respond: echoScript }) }, ctx);
onTestFinished(() => harness.close(ctx));
harness.resume();
const root = await harness.root(ctx);
assert.equal((await root.context(ctx)).messages[0]?.content, "newest edit");
const watch = await root.watch(ctx);
assert.deepEqual(
watch.view.entries.map((entry) => entry.edits),
[
undefined,
[{ target: 2, action: "replace", messages: [{ role: "user", content: "older edit", timestamp: 2 }] }],
[{ target: 2, action: "replace", messages: [{ role: "user", content: "newest edit", timestamp: 3 }] }],
],
);
watch.stop();
});
test("open-time orphaning affects unknown live tasks but never rewrites an already-terminal unknown outcome", async () => {
const storage = new MemoryStorage();
await storage.commit(
[
{ type: "conversation", conversation: { id: 1 } },
{ type: "doc", ref: { doc: "rewindable", conversationId: 1 }, ops: [["r", { plugins: {} }]] },
{
type: "doc",
ref: { doc: "sticky", conversationId: 1 },
ops: [["r", { inbox: [], turn: { tools: [] }, tasks: {}, plugins: {} }]],
},
{
type: "task",
task: {
id: 2,
conversationId: 1,
kind: "removed.kind",
input: null,
status: "running",
after: [],
owns: [],
},
},
{
type: "task",
task: {
id: 3,
conversationId: 1,
kind: "removed.kind",
input: null,
status: "terminal",
outcome: { status: "completed", result: { retained: true } },
after: [],
owns: [],
},
},
],
ctx,
);
const harness = await Harness.open(storage, { models: fake({ respond: echoScript }) }, ctx);
onTestFinished(() => harness.close(ctx));
harness.resume();
assert.deepEqual((await harness.waitForTask(2, ctx)).outcome, { status: "orphaned" });
assert.deepEqual((await harness.getTask(3, ctx))?.outcome, { status: "completed", result: { retained: true } });
});
test("terminal outcomes remain dependency evidence after retirement and JSONL reopen", async () => {
const kind = defineTask<{ value: number }, { phase: "finish" }, { value: number }, null, null>({
name: "spec.retained.dependency",
async initial() {
return { next: { phase: "finish" } };
},
phases: {
async finish(task) {
return { done: () => ({ status: "completed", result: { value: task.input.value } }) };
},
},
async abort() {
return () => null;
},
});
let env = await open({ backend: "jsonl", taskKinds: [kind] });
onTestFinished(() => env.close());
const first = await env.root.commit(
(tx) => tx.createTask(kind, { value: 1 }, { conversationId: 1, background: true }),
ctx,
);
await env.h.waitForTask(first.id, ctx);
for (let value = 2; value <= 30; value++) {
const ref = await env.root.commit(
(tx) => tx.createTask(kind, { value }, { conversationId: 1, background: true, after: [first.id] }),
ctx,
);
await env.h.waitForTask(ref.id, ctx);
}
await env.crash();
env = await open({ dir: env.dir, backend: "jsonl", taskKinds: [kind] });
const afterReopen = await env.root.commit(
(tx) => tx.createTask(kind, { value: 31 }, { conversationId: 1, background: true, after: [first.id] }),
ctx,
);
assert.deepEqual((await env.h.waitForTask(afterReopen.id, ctx)).outcome, {
status: "completed",
result: { value: 31 },
});
assert.deepEqual((await env.h.getTask(first.id, ctx))?.outcome, {
status: "completed",
result: { value: 1 },
});
});

View file

@ -0,0 +1,301 @@
import assert from "node:assert/strict";
import { withAbortSignal } from "@earendil-works/chord/context";
import { onTestFinished, test } from "vitest";
import { Harness, kinds } from "../../../src/harness/pico3/harness.ts";
import { MemoryStorage } from "../../../src/harness/pico3/memory.ts";
import {
defineTask,
type KindConfig,
ReadAfterWrite,
type Storage,
type Write,
} from "../../../src/harness/pico3/types.ts";
import { ctx, echoScript, fake, Gate, open } from "./helpers.ts";
test("direct table reads and new-conversation document snapshots observe all same-batch creations", async () => {
const env = await open({});
onTestFinished(() => env.close());
const observed = await env.root.commit(async (tx) => {
const inputId = await tx.write(1, { kind: "same-batch.entry", data: { value: 1 } });
const input = await tx.input(inputId);
const entry = await tx.entry(input!.entry!);
const entries = await tx.entries([input!.entry!]);
const task = tx.createTask(
kinds.plugin,
{ handler: "missing", input: null },
{ conversationId: 1, background: true },
);
const conversationId = tx.createConversation({
rewindable: { profile: "child-profile" },
sticky: { followUpMode: "all" },
});
const conversation = await tx.conversation(conversationId);
const taskRecord = await tx.task(task);
return {
conversationId,
input,
entry,
entries: [...entries.values()],
taskRecord,
conversation,
rewindable: tx.snapshot({ doc: "rewindable", conversationId }),
sticky: tx.snapshot({ doc: "sticky", conversationId }),
};
}, ctx);
assert.equal(observed.input?.status, "done");
assert.deepEqual(observed.entry?.data, { value: 1 });
assert.equal(observed.entries[0]?.id, observed.entry?.id);
assert.equal(observed.taskRecord?.status, "pending");
assert.equal(observed.conversation?.id, observed.conversationId);
assert.equal(observed.rewindable.profile, "child-profile");
assert.equal(observed.sticky.followUpMode, "all");
});
test("task patch and slot overlays are read-your-writes within one runtime commit", async () => {
let observed: unknown;
const kind = defineTask<null, { phase: "finish" }, null, null, null, object, KindConfig, { count: number }>({
name: "spec.overlay.task",
slot: () => ({ count: 0 }),
async initial(_task, runtime, context) {
observed = await runtime.commit(async (tx, current) => {
tx.checkpoint({ phase: "finish" });
const ref = { id: current.id, kind };
const slot = tx.slot(ref);
slot.count++;
const direct = await tx.task(current.id);
const typed = await tx.task(ref);
const snapshot = tx.snapshot({ doc: "sticky", conversationId: current.conversationId });
return {
directPhase: direct?.checkpoint?.phase,
typedPhase: typed?.checkpoint?.phase,
slotCount: snapshot.tasks[current.id]?.count,
};
}, context);
return { done: () => ({ status: "completed", result: null }) };
},
phases: {
async finish() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env = await open({ taskKinds: [kind] });
onTestFinished(() => env.close());
const ref = await env.root.commit((tx) => tx.createTask(kind, null, { conversationId: 1, background: true }), ctx);
await env.h.waitForTask(ref.id, ctx);
assert.deepEqual(observed, { directPhase: "finish", typedPhase: "finish", slotCount: 1 });
});
test("checkpoint after same-commit owned conversation creation preserves the owns overlay", async () => {
const gate = new Gate();
const kind = defineTask<null, { phase: "finish"; child: number }, null, null, null>({
name: "spec.overlay.owns",
async initial() {
return {
next: (tx) => {
const child = tx.createConversation({});
tx.checkpoint({ phase: "finish", child });
return { phase: "finish", child };
},
};
},
phases: {
async finish(_task, _runtime, context) {
await gate.wait(context);
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
let env = await open({ backend: "jsonl", taskKinds: [kind] });
onTestFinished(() => env.close());
const ref = await env.root.commit(
(tx) => tx.createTask(kind, null, { conversationId: env.root.id, background: true }),
ctx,
);
await gate.arrivals(1);
const running = await env.h.getTask(ref.id, ctx);
assert.deepEqual(running?.owns, [(running?.checkpoint as unknown as { child: number }).child]);
gate.open();
const terminal = await env.h.waitForTask(ref.id, ctx);
assert.deepEqual(terminal.owns, running?.owns);
const expected = structuredClone(terminal);
await env.crash();
env = await open({ backend: "jsonl", dir: env.dir, taskKinds: [kind] });
assert.deepEqual(await env.h.getTask(ref.id, ctx), expected);
});
test("document reads keep a complete overlay while scan-shaped reads reject only affected domains", async () => {
const env = await open({});
onTestFinished(() => env.close());
const state = env.h.namespace<{ nested: { value: number } }>("test.transaction-overlay", {
rewindable: { nested: { value: 0 } },
});
const other = await env.h.createConversation({}, ctx);
await env.root.commit(async (tx) => {
tx.plugins(state).nested = { value: 1 };
tx.config(1).set("profile", "updated");
assert.equal(tx.config(1).get("profile"), "updated");
assert.deepEqual(tx.snapshot({ doc: "rewindable", conversationId: 1 }).plugins["test.transaction-overlay"], {
nested: { value: 1 },
});
await tx.write(1, { kind: "root.write" });
await tx.context(other.id);
await tx.newestEntry(other.id);
await tx.scanEntries({ conversationId: other.id, limit: 10 });
}, ctx);
await assert.rejects(
env.root.commit(async (tx) => {
await tx.write(1, { kind: "root.write.two" });
await tx.context(1);
}, ctx),
ReadAfterWrite,
);
await assert.rejects(
env.root.commit(async (tx) => {
tx.createTask(kinds.plugin, { handler: "missing", input: null }, { conversationId: 1, background: true });
await tx.tasks({ conversationId: other.id });
}, ctx),
ReadAfterWrite,
);
});
test("a caught ReadAfterWrite poisons all buffered table and document writes and the next transaction starts clean", async () => {
const env = await open({});
onTestFinished(() => env.close());
const state = env.h.namespace<{ poisoned: boolean; alsoPoisoned: boolean; clean: boolean }>(
"test.transaction-poison",
{ rewindable: { alsoPoisoned: false }, sticky: { poisoned: false, clean: false } },
);
let caught: unknown;
await assert.rejects(
env.root.commit(async (tx) => {
tx.plugins(state).poisoned = true;
await tx.write(1, { kind: "must.rollback" });
try {
await tx.scanEntries({ conversationId: 1, limit: 10 });
} catch (error) {
caught = error;
}
tx.plugins(state).alsoPoisoned = true;
return "callback returned normally";
}, ctx),
ReadAfterWrite,
);
assert.ok(caught instanceof ReadAfterWrite);
assert.equal((await env.root.sticky(ctx)).plugins["test.transaction-poison"], undefined);
assert.equal((await env.root.rewindable(ctx)).plugins["test.transaction-poison"], undefined);
assert.equal(
(await env.entries()).some((entry) => entry.kind === "must.rollback"),
false,
);
await env.root.commit((tx) => {
tx.plugins(state).clean = true;
}, ctx);
assert.deepEqual((await env.root.sticky(ctx)).plugins["test.transaction-poison"], {
poisoned: false,
clean: true,
});
});
test("a queued passive write does not poison entry scans because it has not changed the entry domain", async () => {
const providerGate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate: providerGate }) });
onTestFinished(() => env.close());
const active = await env.root.send({ content: "active" }, ctx);
await providerGate.arrivals(1);
const result = await env.root.commit(async (tx) => {
const queued = await tx.write(1, { kind: "queued.passive" });
const input = await tx.input(queued);
const context = await tx.context(1);
const newest = await tx.newestEntry(1);
return { queued, input, contextIds: context.entries.map((entry) => entry.id), newest: newest?.id };
}, ctx);
assert.equal(result.input?.status, "queued");
assert.ok(result.contextIds.includes(result.newest!));
providerGate.open();
await active.wait(ctx);
assert.equal((await env.input(result.queued))?.status, "done");
});
test("callback-minted but uncommitted ids may be reused only after JSONL reopen", async () => {
let env = await open({ backend: "jsonl" });
onTestFinished(() => env.close());
let abandoned = 0;
await assert.rejects(
env.root.commit(async (tx) => {
abandoned = await tx.write(1, { kind: "abandoned" });
throw new Error("discard transaction");
}, ctx),
/discard transaction/,
);
const sameProcess = await env.root.write({ kind: "same-process" }, ctx);
assert.ok(sameProcess > abandoned, "the live storage instance does not recycle a minted id");
await env.crash();
env = await open({ dir: env.dir, backend: "jsonl" });
const afterReopen = await env.root.write({ kind: "after-reopen" }, ctx);
assert.ok(afterReopen > sameProcess, "committed ids remain above high-water after reopen");
let secondAbandoned = 0;
await assert.rejects(
env.root.commit(async (tx) => {
secondAbandoned = await tx.write(1, { kind: "abandoned.two" });
throw new Error("discard again");
}, ctx),
/discard again/,
);
await env.crash();
env = await open({ dir: env.dir, backend: "jsonl" });
assert.equal(await env.root.write({ kind: "reuses-uncommitted" }, ctx), secondAbandoned);
});
test("once persistence starts it is attempted exactly once with a non-cancellable context", async () => {
class BlockingStorage extends MemoryStorage {
block = false;
attempts = 0;
resolveStarted: (() => void) | undefined;
readonly started = new Promise<void>((resolve) => {
this.resolveStarted = resolve;
});
resolveRelease: (() => void) | undefined;
readonly release = new Promise<void>((resolve) => {
this.resolveRelease = resolve;
});
override async commit(writes: readonly Write[], context: Parameters<Storage["commit"]>[1]) {
if (this.block) {
this.attempts++;
assert.equal(context.abortSignal, undefined);
this.resolveStarted!();
await this.release;
}
return super.commit(writes, context);
}
}
const storage = new BlockingStorage();
const harness = await Harness.open(storage, { models: fake({ respond: echoScript }) }, ctx);
onTestFinished(() => harness.close(ctx));
harness.resume();
const root = await harness.root(ctx);
storage.block = true;
const controller = new AbortController();
const operationContext = withAbortSignal(controller.signal, ctx);
const write = root.write({ kind: "persist-despite-caller-cancel" }, operationContext);
await storage.started;
controller.abort(new Error("too late"));
storage.resolveRelease!();
const id = await write;
assert.ok(id > 0);
assert.equal(storage.attempts, 1);
assert.equal(
(await harness.entries({ conversationId: 1, limit: 100 }, ctx)).some(
(entry) => entry.kind === "persist-despite-caller-cancel",
),
true,
);
});

View file

@ -0,0 +1,280 @@
import assert from "node:assert/strict";
import type { Context } from "@earendil-works/chord";
import type { AssistantMessage, AssistantMessageEvent } from "@earendil-works/pi-ai";
import { onTestFinished, test } from "vitest";
import type { ConversationHandle } from "../../../src/harness/pico3/harness.ts";
import type { ConversationView, Envelope, Models, ViewEvent } from "../../../src/harness/pico3/types.ts";
import { ctx, fake, Gate, model, open } from "./helpers.ts";
type SpecWatch = {
readonly view: ConversationView;
readonly revision: number;
readonly closed: boolean;
start(listener: (envelope: Envelope) => void): void;
stop(): void;
};
const specWatch = (conversation: Pick<ConversationHandle<object>, "watch">) =>
conversation.watch(ctx) as unknown as Promise<SpecWatch>;
const usage = (): AssistantMessage["usage"] => ({
input: 1,
output: 1,
cacheRead: 0,
cacheWrite: 0,
totalTokens: 2,
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
});
function partialModels(gate: Gate): Models {
const resolved = {
id: "fake-1",
name: "Fake",
api: "anthropic-messages" as const,
provider: "anthropic" as const,
baseUrl: "",
reasoning: false,
input: ["text" as const],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 200_000,
maxTokens: 8192,
};
return {
resolve: () => resolved,
async *stream(_model, _request, context: Context): AsyncIterable<AssistantMessageEvent> {
const partial: AssistantMessage = {
role: "assistant",
content: [],
api: resolved.api,
provider: resolved.provider,
model: resolved.id,
usage: usage(),
stopReason: "stop",
timestamp: 1,
};
yield { type: "start", partial };
partial.content.push({ type: "text", text: "" });
yield { type: "text_start", contentIndex: 0, partial };
(partial.content[0] as { text: string }).text = "partial";
yield { type: "text_delta", contentIndex: 0, delta: "partial", partial };
await gate.wait(context);
yield { type: "text_end", contentIndex: 0, content: "partial", partial };
yield { type: "done", reason: "stop", message: { ...partial, stopReason: "stop" } };
},
async fetchDeferred() {
throw new Error("not deferred");
},
async cancelDeferred() {},
};
}
test("watch snapshot is the single flat rendering document and excludes raw documents, records, checkpoints, and private slot state", async () => {
const env = await open({});
onTestFinished(() => env.close());
const watch = await specWatch(env.root);
assert.deepEqual(
Object.keys(watch.view).sort(),
["compaction", "config", "conversation", "entries", "inbox", "plugins", "tasks", "turn"]
.filter((key) => key in watch.view)
.sort(),
);
assert.ok("config" in watch.view);
assert.ok("inbox" in watch.view);
assert.ok("plugins" in watch.view);
assert.ok(!("rewindable" in (watch.view as object)));
assert.ok(!("sticky" in (watch.view as object)));
for (const task of Object.values(watch.view.tasks)) {
assert.ok(!("checkpoint" in task));
assert.ok(!("outcome" in task));
assert.ok(!("memos" in task));
}
watch.stop();
});
test("watch revisions are contiguous per watch even when unrelated conversation commits create global storage gaps", async () => {
const env = await open({});
onTestFinished(() => env.close());
const other = await env.h.createConversation({}, ctx);
const watch = await specWatch(env.root);
const revisions: number[] = [];
watch.start((envelope) => revisions.push(envelope.revision));
await env.root.write({ kind: "root.one" }, ctx);
await other.write({ kind: "other.only" }, ctx);
await env.root.write({ kind: "root.two" }, ctx);
assert.deepEqual(revisions, [watch.revision + 1, watch.revision + 2]);
watch.stop();
});
test("one admission commit produces one atomic op envelope with entry/input/turn events", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: () => ({ text: "answer" }), gate }) });
onTestFinished(() => env.close());
const watch = await specWatch(env.root);
const envelopes: Envelope[] = [];
watch.start((envelope) => envelopes.push(envelope));
const input = await env.root.send({ content: "hello" }, ctx);
await gate.arrivals(1);
const admissions = envelopes.filter((envelope) => envelope.events.some((event) => event.type === "turn.started"));
assert.equal(admissions.length, 1, "send has exactly one atomic admission envelope");
const admission = admissions[0]!;
assert.ok(admission.ops.some((op) => op[0] === "p" && op[1]?.[0] === "entries"));
assert.ok(admission.ops.some((op) => op[0] === "s" && op[1]?.[0] === "turn"));
assert.deepEqual(
admission.events.map((event) => event.type),
["input.placed", "entry.added", "turn.started"],
);
gate.open();
await input.wait(ctx);
watch.stop();
});
test("terminal answer entry, input settlement, generation completion, and turn end share one envelope", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: () => ({ text: "answer" }), gate }) });
onTestFinished(() => env.close());
const watch = await specWatch(env.root);
const envelopes: Envelope[] = [];
watch.start((envelope) => envelopes.push(envelope));
const input = await env.root.send({ content: "hello" }, ctx);
await gate.arrivals(1);
gate.open();
const result = await input.wait(ctx);
const terminal = envelopes.find((envelope) => envelope.events.some((event) => event.type === "turn.ended"));
assert.ok(terminal);
assert.ok(terminal.events.some((event) => event.type === "generation.completed"));
assert.ok(
terminal.events.some(
(event) => event.type === "turn.ended" && event.status === "done" && event.answer === result.answer,
),
);
assert.ok(terminal.events.some((event) => event.type === "entry.added" && event.entry.id === result.answer));
assert.ok(terminal.ops.some((op) => op[0] === "d" && op[1]?.[0] === "turn"));
watch.stop();
});
test("late joiner reconstructs streaming generation state without replayed events", async () => {
const gate = new Gate();
const env = await open({ models: partialModels(gate), root: { rewindable: { model } } });
onTestFinished(() => env.close());
const input = await env.root.send({ content: "stream" }, ctx);
await gate.arrivals(1);
const watch = await specWatch(env.root);
assert.equal(watch.view.turn?.generation?.stage, "streaming");
assert.equal((watch.view.turn?.message?.content[0] as { text?: string } | undefined)?.text, "partial");
const seen: Envelope[] = [];
watch.start((envelope) => seen.push(envelope));
assert.deepEqual(seen, [], "capture does not replay historical events");
gate.open();
await input.wait(ctx);
assert.ok(seen.length > 0);
watch.stop();
});
test("a head commit is represented by transcript splice ops and matching entry/head events", async () => {
const env = await open({});
onTestFinished(() => env.close());
await env.root.write({ kind: "before" }, ctx);
const watch = await specWatch(env.root);
let envelope: Envelope | undefined;
watch.start((next) => {
envelope = next;
});
await env.root.reset("fresh", ctx);
assert.ok(envelope);
const entryOps = envelope.ops.filter((op) => op[0] === "p" && op[1]?.[0] === "entries");
assert.equal(entryOps.length, 2);
assert.equal(entryOps[0]?.[2], 0);
assert.ok((entryOps[0]?.[3] as number) > 0);
assert.deepEqual(
envelope.events.map((event) => event.type),
["head.moved", "entry.added"],
);
watch.stop();
});
test("a namespace projection failure closes its watches without failing the persisted writer", async () => {
const reports: unknown[] = [];
const env = await open({ onReport: (error) => reports.push(error) });
onTestFinished(() => env.close());
const namespace = env.h.namespace<{ fail: boolean }>(
"spec.bad-projection",
{ sticky: { fail: false } },
{
view: (slice) => {
if (slice.fail) throw new Error("projection failed");
return { ok: true };
},
},
);
const watch = await env.root.watch(ctx);
watch.start(() => {});
await env.root.commit((tx) => {
tx.plugins(namespace).fail = true;
}, ctx);
assert.equal(watch.closed, true);
assert.match(String(reports[0]), /projection failed/);
assert.equal((await env.root.sticky(ctx)).plugins[namespace.id]?.fail, true);
});
test("all watchers of one conversation receive the same commit envelope object while retaining independent lifecycle", async () => {
const env = await open({});
onTestFinished(() => env.close());
const first = await specWatch(env.root);
const second = await specWatch(env.root);
let a: Envelope | undefined;
let b: Envelope | undefined;
first.start((envelope) => {
a = envelope;
});
second.start((envelope) => {
b = envelope;
});
await env.root.write({ kind: "shared" }, ctx);
assert.ok(a && b);
assert.equal(a, b, "fan-out reuses the one envelope assembled for the conversation commit");
first.stop();
await env.root.write({ kind: "second-only" }, ctx);
assert.equal(first.closed, true);
assert.equal(second.closed, false);
second.stop();
});
test("pre-start delivery is ordered, start is idempotent, and stop has a hard no-callback boundary", async () => {
const env = await open({});
onTestFinished(() => env.close());
const watch = await specWatch(env.root);
await env.root.write({ kind: "buffered.one" }, ctx);
await env.root.write({ kind: "buffered.two" }, ctx);
const received: Envelope[] = [];
watch.start((envelope) => received.push(envelope));
watch.start(() => {
throw new Error("second start must be ignored");
});
assert.equal(received.length, 2);
assert.equal(received[1]!.revision, received[0]!.revision + 1);
const countAtStop = received.length;
watch.stop();
watch.stop();
await env.root.write({ kind: "after.stop" }, ctx);
assert.equal(received.length, countAtStop);
});
test("listener failure is isolated from persistence and from sibling watches", async () => {
const reports: unknown[] = [];
const env = await open({ onReport: (error) => reports.push(error) });
onTestFinished(() => env.close());
const broken = await specWatch(env.root);
const healthy = await specWatch(env.root);
const healthyEvents: ViewEvent[][] = [];
broken.start(() => {
throw new Error("listener failed");
});
healthy.start((envelope) => healthyEvents.push(envelope.events));
const persisted = await env.root.write({ kind: "survives.listener" }, ctx);
assert.ok(persisted > 0);
assert.equal(broken.closed, true);
assert.equal(healthy.closed, false);
assert.equal(healthyEvents.length, 1);
assert.match(String(reports[0]), /listener failed/);
healthy.stop();
});

View file

@ -0,0 +1,267 @@
import assert from "node:assert/strict";
import { Type } from "typebox";
import { onTestFinished, test } from "vitest";
import { kinds } from "../../../src/harness/pico3/harness.ts";
import type { RequestMessage, ToolDeclaration } from "../../../src/harness/pico3/types.ts";
import {
contentOf,
ctx,
echoScript,
failureOf,
fake,
Gate,
kinds as kindsOf,
lastMessage,
model,
open,
untilTerminal,
} from "./helpers.ts";
const delegate: ToolDeclaration = {
name: "delegate",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute(args, api, ctx) {
const child = await api.conversation(
{
rewindable: { model, selectedTools: [], plugins: { forged: { child: true } } },
sticky: { plugins: { forged: { child: true } } },
},
ctx,
);
const sent = await child.send({ content: `child task ${(args as { v: string }).v}` }, ctx);
const r = await sent.wait(ctx);
return {
content: [{ type: "text", text: `child ${child.id} ${r.status}` }],
details: { child: child.id, ...(r.answer === undefined ? {} : { answer: r.answer }) },
};
},
};
test("subagent: owned conversation, isolated config, answers, parent continues", async () => {
const env = await open({
tools: [delegate],
models: fake({
respond: (m) => {
const last = lastMessage(m);
if (last.role === "user" && String((last as { content: string }).content).startsWith("child task"))
return { text: "child says hi" };
return echoScript(m);
},
}),
});
onTestFinished(() => env.close());
const a = await (await env.root.send({ content: "tool:delegate" }, ctx)).wait(ctx);
await env.root.waitForIdle(ctx);
assert.equal(a.status, "done");
const tr = (await env.entries()).find((e) => e.kind === "pi.tool_result")!;
const { child, answer } = (tr.data as { details: { child: number; answer: number } }).details;
assert.match(contentOf(tr), /done/);
const conv = (await env.root.commit((tx) => tx.conversation(child), ctx))!;
const toolTask = (await env.tasks(1)).find((t) => t.kind === "pi.tool")!;
assert.equal(conv.owner, toolTask.id);
assert.deepEqual(toolTask.owns, [child]);
assert.equal(conv.parent, undefined);
assert.equal(kindsOf(await env.entries(child)), "user system assistant");
assert.equal(
contentOf((await env.entries(child)).find((e) => e.id === answer)),
JSON.stringify([{ type: "text", text: "child says hi " }]),
);
const cr = await (await env.h.conversation(child, ctx))!.rewindable(ctx);
assert.deepEqual(cr.plugins, {});
assert.deepEqual((await (await env.h.conversation(child, ctx))!.sticky(ctx)).plugins, {});
assert.deepEqual(cr.selectedTools, []);
// subtree hooks reach the child
});
test("conversation abort reaches an owned child mid-stream; background children survive", async () => {
const gate = new Gate();
const isChild = (m: RequestMessage[]) => {
const last = lastMessage(m);
return last.role === "user" && String((last as { content: string }).content).startsWith("child task");
};
const env = await open({
tools: [delegate],
models: fake({ respond: (m) => (isChild(m) ? { text: "slow child" } : echoScript(m)), gate, gateWhen: isChild }),
});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "tool:delegate" }, ctx);
await gate.arrivals(1); // the child's generation is blocked at the gate
const live = (await env.tasks()).filter((t) => t.status !== "terminal");
const childConv = live.find((t) => t.conversationId !== 1)!.conversationId;
// a background job in the child survives conversation abort
const bg = await env.root.commit(
(tx) =>
tx.createTask(kinds.plugin, { handler: "x", input: null }, { conversationId: childConv, background: true }).id,
ctx,
);
await env.root.abort(ctx);
assert.equal((await env.input(a.id))?.reason, "aborted");
const after = await env.tasks();
assert.equal(after.find((t) => t.kind === "pi.tool")!.outcome?.status, "aborted");
const childGen = after.find((t) => t.conversationId === childConv && t.kind === "pi.generation")!;
assert.equal(childGen.outcome?.status, "aborted");
assert.ok(
!(await env.entries(childConv)).some((e) => e.kind === "pi.assistant"),
"gated before the first token: no partial, so no aborted assistant entry",
);
// the bg plugin task ran (no handler → failed), was not aborted
assert.equal(failureOf(await untilTerminal(env, bg))?.reason, "missing_handler");
assert.equal((await env.tasks()).filter((t) => t.status !== "terminal").length, 0);
});
test("subtree hooks: registered on the parent with subtree:true fire for the child", async () => {
const seen: number[] = [];
const env = await open({
tools: [delegate],
models: fake({
respond: (m) => {
const last = lastMessage(m);
if (last.role === "user" && String((last as { content: string }).content).startsWith("child task"))
return { text: "ok" };
return echoScript(m);
},
}),
});
onTestFinished(() => env.close());
const hookNamespace = env.h.namespace("test.subtree-hooks", {});
env.root.hooks(
hookNamespace,
kinds.generation,
{
afterResponse: (_m, info) => {
seen.push(info.conversationId);
},
},
{ subtree: true },
);
await (await env.root.send({ content: "tool:delegate" }, ctx)).wait(ctx);
await env.root.waitForIdle(ctx);
assert.ok(seen.includes(1));
assert.ok(
seen.some((c) => c !== 1),
`child response observed: ${seen}`,
);
// without subtree, only the parent
const seen2: number[] = [];
env.root.hooks(hookNamespace, kinds.generation, {
afterResponse: (_m, info) => {
seen2.push(info.conversationId);
},
});
await (await env.root.send({ content: "tool:delegate" }, ctx)).wait(ctx);
await env.root.waitForIdle(ctx);
assert.deepEqual([...new Set(seen2)], [1]);
});
test("subtree hooks after reopen: ancestry is reconstructed from conversation owners (including a terminal owner task), not by matching ids", async () => {
const respond = (m: RequestMessage[]) => {
const last = lastMessage(m);
if (last.role === "user" && String((last as { content: string }).content).startsWith("child task"))
return { text: "ok" };
return echoScript(m);
};
let env = await open({ backend: "jsonl", tools: [delegate], models: fake({ respond }) });
onTestFinished(() => env.close());
await (await env.root.send({ content: "tool:delegate" }, ctx)).wait(ctx);
await env.root.waitForIdle(ctx);
const child = (await env.root.commit((tx) => tx.tasks({}), ctx)).find((t) => t.kind === "pi.tool")!.owns[0]!;
assert.ok(child !== undefined);
await env.crash();
env = await open({ dir: env.dir, backend: "jsonl", tools: [delegate], models: fake({ respond }) });
const hookNamespace = env.h.namespace("test.reopened-subtree-hooks", {});
const seen: number[] = [];
env.root.hooks(
hookNamespace,
kinds.generation,
{
afterResponse: (_m, info) => {
seen.push(info.conversationId);
},
},
{ subtree: true },
);
const c = (await env.h.conversation(child, ctx))!;
await (await c.send({ content: "child task again" }, ctx)).wait(ctx); // the owner tool task is long terminal
assert.deepEqual(seen, [child]);
const seen2: number[] = [];
env.root.hooks(hookNamespace, kinds.generation, {
afterResponse: (_m, info) => {
seen2.push(info.conversationId);
},
});
await (await c.send({ content: "child task once more" }, ctx)).wait(ctx);
assert.deepEqual(seen2, []); // no subtree flag: the child is not the parent
});
test("task abort (not conversation abort) keeps queued input; a later idle send consumes it", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
await gate.arrivals(1);
const f = await env.root.send({ content: "F" }, ctx);
const gen = (await env.tasks()).find((t) => t.kind === "pi.generation")!;
// abort the task through a plugin (the ToolApi has no abortTask; use a host commit + scheduler via harness.abortTask? not exposed) → mark + let the scheduler run fresh abort
await env.h.markTask(gen.id, ctx);
await untilTerminal(env, gen.id);
assert.equal((await env.input(a.id))?.reason, "aborted");
assert.equal((await env.input(f.id))?.status, "queued"); // survives task abort
gate.open();
const b = await env.root.send({ content: "B" }, ctx);
await b.wait(ctx);
await env.root.waitForIdle(ctx);
assert.equal((await env.input(f.id))?.status, "done"); // placed by the idle send, before B
const users = (await env.entries()).filter((e) => e.kind === "pi.user").map(contentOf);
assert.deepEqual(users, ["A", "F", "B"]);
});
test("api.conversation({ inherit: true }) forks the tool's own conversation at its tip: transcript and rewindable inherited", async () => {
const inheriting: ToolDeclaration = {
name: "inherit",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute(_a, api, ctx) {
const child = await api.conversation({ inherit: true, rewindable: { selectedTools: [] } }, ctx);
const r = await (await child.send({ content: "child task: what did we say?" }, ctx)).wait(ctx);
return { content: [{ type: "text", text: `${child.id}:${r.status}` }], details: { child: child.id } };
},
};
const env = await open({
tools: [inheriting],
models: fake({
respond: (m) => {
const last = lastMessage(m);
if (last.role === "user" && String((last as { content: string }).content).startsWith("child task"))
return { text: "we said hi" };
return echoScript(m);
},
}),
});
onTestFinished(() => env.close());
const state = env.h.namespace<{ marker: string }>("test.subagent-state", {
rewindable: { marker: "" },
});
await env.root.commit((tx) => {
tx.plugins(state).marker = "before";
}, ctx);
await (await env.root.send({ content: "hi" }, ctx)).wait(ctx);
await env.root.commit((tx) => {
tx.plugins(state).marker = "at-fork";
}, ctx);
await (await env.root.send({ content: "tool:inherit" }, ctx)).wait(ctx);
await env.root.waitForIdle(ctx);
const tr = (await env.entries()).find((e) => e.kind === "pi.tool_result")!;
const child = (tr.data as { details: { child: number } }).details.child;
const conv = (await env.root.commit((tx) => tx.conversation(child), ctx))!;
assert.equal(conv.parent?.conversationId, 1);
assert.equal(
(await (await env.h.conversation(child, ctx))!.rewindable(ctx)).plugins["test.subagent-state"]?.marker,
"at-fork",
);
assert.deepEqual((await (await env.h.conversation(child, ctx))!.rewindable(ctx)).selectedTools, []);
const { entries } = await (await env.h.conversation(child, ctx))!.commit((tx) => tx.context(child), ctx);
// inherited: user hi, system, assistant, user tool:inherit, system?, assistant(toolcall) … then the child's own user + assistant
assert.ok(kindsOf(entries).startsWith("user system assistant user"));
assert.ok(kindsOf(entries).endsWith("user system assistant") || kindsOf(entries).endsWith("user assistant"));
});

View file

@ -0,0 +1,124 @@
import assert from "node:assert/strict";
import { Type } from "typebox";
import { onTestFinished, test } from "vitest";
import { Bounded } from "../../../src/harness/pico3/bounded.ts";
import { Forbidden, type ToolDeclaration, type ToolSlot } from "../../../src/harness/pico3/types.ts";
import { ctx, open } from "./helpers.ts";
test("Bounded: a single chunk larger than maxBytes is sliced on push and counted as dropped; line caps apply to head and tail", () => {
const tail = new Bounded(10, 100, "tail");
tail.push(new TextEncoder().encode("0123456789ABCDEFGHIJ")); // 20 bytes
assert.equal(tail.text(), "ABCDEFGHIJ");
assert.equal(tail.droppedBytes, 10);
const head = new Bounded(10, 100, "head");
head.push(new TextEncoder().encode("0123456789ABCDEFGHIJ"));
assert.equal(head.text(), "0123456789");
assert.equal(head.droppedBytes, 10);
const lines = new Bounded(1000, 2, "head");
lines.push(new TextEncoder().encode("a\nb\nc\nd\n"));
assert.equal(lines.text(), "a\nb\n");
assert.equal(lines.droppedLines, 2);
const tlines = new Bounded(1000, 2, "tail");
for (const l of ["a\n", "b\n", "c\n", "d\n"]) tlines.push(new TextEncoder().encode(l));
assert.equal(tlines.text(), "c\nd\n");
assert.equal(tlines.droppedLines, 2);
});
test("tool streaming: byte and line bounds are enforced on the stream and on the stored result; progress cannot touch identity fields", async () => {
const schema = Type.Object({ v: Type.String() });
let forbidden: unknown;
const t: ToolDeclaration<typeof schema> = {
name: "x",
description: "",
parameters: schema,
output: { maxBytes: 20, maxLines: 3, retain: "tail" },
async execute(_a, api, c) {
for (let i = 0; i < 10; i++) api.stream(`line ${i}\n`);
await api.progress((s) => {
(s as ToolSlot).callId = "forged";
(s as ToolSlot).name = "forged";
(s as ToolSlot).output = "forged";
s.progress = "p";
}, c);
try {
await api.progress((s) => {
(s as { args?: unknown }).args = 1;
}, c);
} catch (e) {
forbidden = e;
}
return {};
},
};
const env = await open({ tools: [t] });
onTestFinished(() => env.close());
await (await env.root.send({ content: "tool:x" }, ctx)).wait(ctx);
const result = (await env.entries()).find((e) => e.kind === "pi.tool_result")!;
const text = (result.model![0] as { content: { text: string }[] }).content[0]!.text;
assert.ok(new TextEncoder().encode(text).length <= 20, text);
assert.ok(text.split("\n").length <= 4);
assert.ok(text.endsWith("line 9\n"));
const data = result.data as { truncated?: { bytes: number; lines: number }; diagnostics: { code: string }[] };
assert.ok(data.truncated!.bytes > 0 && data.truncated!.lines > 0);
assert.equal(data.diagnostics[0]!.code, "truncated");
void forbidden; // identity fields are copied into a free-field view; assigning them is simply ignored
assert.equal(JSON.stringify((await env.root.sticky(ctx)).turn), JSON.stringify({ tools: [] }));
});
test("returned text blocks share one aggregate output budget", async () => {
const declaration: ToolDeclaration = {
name: "aggregate",
description: "",
parameters: Type.Object({ v: Type.String() }),
output: { maxBytes: 10, maxLines: 100, retain: "tail" },
async execute() {
return {
content: [
{ type: "text", text: "abcdefgh" },
{ type: "text", text: "ijklmnop" },
{ type: "text", text: "qrstuvwx" },
],
};
},
};
const env = await open({ tools: [declaration] });
onTestFinished(() => env.close());
await (await env.root.send({ content: "tool:aggregate" }, ctx)).wait(ctx);
const result = (await env.entries()).find((entry) => entry.kind === "pi.tool_result")!;
const content = (result.model![0] as { content: Array<{ type: string; text?: string }> }).content;
const texts = content.filter((block) => block.type === "text");
assert.deepEqual(texts, [{ type: "text", text: "opqrstuvwx" }]);
assert.ok(new TextEncoder().encode(texts.map((block) => block.text).join("")).length <= 10);
assert.equal((result.data as { truncated: { bytes: number } }).truncated.bytes, 14);
});
test("a tool cannot reach core operations through the ToolApi; a hook that throws blocks the call before `started`", async () => {
const schema = Type.Object({ v: Type.String() });
let calls = 0;
const t: ToolDeclaration<typeof schema> = {
name: "x",
description: "",
parameters: schema,
async execute() {
calls++;
return { content: [{ type: "text", text: "ran" }] };
},
};
const env = await open({
tools: [t],
hooks: {
tool: {
beforeTool: () => {
throw new Error("nope");
},
},
},
});
onTestFinished(() => env.close());
await (await env.root.send({ content: "tool:x" }, ctx)).wait(ctx);
assert.equal(calls, 0);
const r = (await env.entries()).find((e) => e.kind === "pi.tool_result")!;
assert.match(JSON.stringify(r.model), /blocked: hook threw/);
assert.equal((await env.tasks()).find((x) => x.kind === "pi.tool")?.checkpoint, undefined);
void Forbidden;
});

View file

@ -0,0 +1,400 @@
import assert from "node:assert/strict";
import { onTestFinished, test } from "vitest";
import { effectiveTools } from "../../../src/harness/pico3/system.ts";
import { contentOf, ctx, echoScript, fake, Gate, kinds, open, sleep, tool } from "./helpers.ts";
for (const backend of ["memory", "jsonl"] as const) {
test(`[${backend}] one turn: user → generation → 2 tools → post_tools → generation → final`, async () => {
const a = tool("a"),
b = tool("b");
const env = await open({ backend, tools: [a, b] });
onTestFinished(() => env.close());
const inp = await env.root.send({ content: "tool:a,b" }, ctx);
const r = await inp.wait(ctx);
await env.root.waitForIdle(ctx);
assert.equal(r.status, "done");
assert.equal(kinds(await env.entries()), "user system assistant tool_result tool_result assistant");
assert.equal(a.calls, 1);
assert.equal(b.calls, 1);
const tasks = await env.tasks();
assert.deepEqual(tasks.map((t) => `${t.kind}:${t.status}:${t.outcome?.status}`).sort(), [
"pi.generation:terminal:completed",
"pi.generation:terminal:completed",
"pi.post_tools:terminal:completed",
"pi.tool:terminal:completed",
"pi.tool:terminal:completed",
]);
const answer = (await env.entries()).find((e) => e.id === r.answer)!;
assert.equal(contentOf(answer), JSON.stringify([{ type: "text", text: "after tools " }]));
// tool results are in call order in the projection
const { messages } = await env.root.commit((tx) => tx.context(1), ctx);
assert.deepEqual(
messages.map((m) => m.role),
["user", "system", "assistant", "toolResult", "toolResult", "assistant"],
); // system is appended at preparation, after the user entry (§12.5)
// turn view cleared
assert.deepEqual((await env.root.sticky(ctx)).turn, { tools: [] });
assert.deepEqual((await env.root.sticky(ctx)).tasks, {});
await env.close();
});
}
test("requestId dedup returns the original input and writes nothing", async () => {
const env = await open({});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "hi", requestId: "k" }, ctx);
const b = await env.root.send({ content: "other", requestId: "k" }, ctx);
assert.equal(a.id, b.id);
await a.wait(ctx);
assert.equal((await env.entries()).filter((e) => e.kind === "pi.user").length, 1);
});
test("whenBusy: followUp queues, steer queues, reject throws; idle send places", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const first = await env.root.send({ content: "A" }, ctx);
await gate.arrivals(1);
const f = await env.root.send({ content: "F" }, ctx);
const s = await env.root.send({ content: "S", whenBusy: "steer" }, ctx);
await assert.rejects(env.root.send({ content: "R", whenBusy: "reject" }, ctx), /busy/i);
assert.equal((await env.input(f.id))?.status, "queued");
assert.equal((await env.input(s.id))?.status, "queued");
const sticky = await env.root.sticky(ctx);
assert.deepEqual(
sticky.inbox.map((q) => q.mode),
["followUp", "steer"],
);
gate.open();
await first.wait(ctx);
await env.root.waitForIdle(ctx);
// A answered; S placed at the final boundary too (no post-tools boundary in a no-tool turn); F placed. Both form the successor group.
assert.equal((await env.input(f.id))?.status, "done");
assert.equal((await env.input(s.id))?.status, "done");
assert.equal(kinds(await env.entries()), "user system assistant user user assistant");
});
test("steer joins the active group at the post-tools boundary; followUp waits for the final answer", async () => {
const gate = new Gate();
const t = tool("a", { gate });
const env = await open({ tools: [t] });
onTestFinished(() => env.close());
const first = await env.root.send({ content: "tool:a" }, ctx);
await gate.arrivals(1); // tool running: generation done, post_tools pending
const s = await env.root.send({ content: "S", whenBusy: "steer" }, ctx);
const f = await env.root.send({ content: "F" }, ctx);
gate.open();
await first.wait(ctx);
await env.root.waitForIdle(ctx);
const entries = await env.entries();
// S placed right after the tool results (before the continuation generation), F after the final answer
assert.equal(kinds(entries), "user system assistant tool_result user assistant user assistant");
const sEntry = (await env.input(s.id))!;
const fEntry = (await env.input(f.id))!;
assert.equal(sEntry.status, "done");
assert.equal(fEntry.status, "done");
assert.ok(sEntry.entry! < fEntry.entry!);
// S's answer is the continuation's answer, which is also A's answer
assert.equal((await env.input(first.id))?.answer, sEntry.answer);
assert.notEqual(sEntry.answer, fEntry.answer);
});
test("one-at-a-time vs all followUp modes", async () => {
for (const mode of ["one-at-a-time", "all"] as const) {
const gate = new Gate();
const env = await open({
models: fake({ respond: echoScript, gate }),
root: { rewindable: { model }, sticky: { followUpMode: mode } },
});
onTestFinished(() => env.close().catch(() => {}));
const a = await env.root.send({ content: "A" }, ctx);
await gate.arrivals(1);
const f1 = await env.root.send({ content: "F1" }, ctx);
const f2 = await env.root.send({ content: "F2" }, ctx);
gate.open();
await a.wait(ctx);
await env.root.waitForIdle(ctx);
const gens = (await env.tasks()).filter((t) => t.kind === "pi.generation");
const secondGroup = (gens[1]!.input as { inputs: number[] }).inputs;
if (mode === "all") assert.deepEqual(secondGroup, [f1.id, f2.id]);
else {
assert.deepEqual(secondGroup, [f1.id]);
assert.deepEqual((gens[2]!.input as { inputs: number[] }).inputs, [f2.id]); // F2 placed at the idle-turn boundary after F1's answer
}
assert.equal((await env.input(f2.id))?.status, "done");
await env.close();
}
});
const model = { provider: "anthropic", modelId: "fake-1" };
test("write: appended when idle, queued when busy, placed at the next boundary, done without an answer", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const w0 = await env.root.write(
{ kind: "pi.notice", model: [{ role: "user", content: "idle note", timestamp: 0 }] },
ctx,
);
assert.equal((await env.input(w0))?.status, "done");
const a = await env.root.send({ content: "A" }, ctx);
await gate.arrivals(1);
const w1 = await env.root.write(
{ kind: "pi.notice", model: [{ role: "user", content: "busy note", timestamp: 0 }] },
ctx,
);
assert.equal((await env.input(w1))?.status, "queued");
gate.open();
await a.wait(ctx);
await env.root.waitForIdle(ctx);
assert.equal((await env.input(w1))?.status, "done");
assert.equal((await env.input(w1))?.answer, undefined);
assert.equal(kinds(await env.entries()), "notice user system assistant notice");
});
test("InputHandle.abort: queued → aborted and removed; placed → already_placed; unknown → not_found", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
await gate.arrivals(1);
const f = await env.root.send({ content: "F" }, ctx);
assert.equal(await f.abort(ctx), "aborted");
assert.equal((await env.input(f.id))?.status, "unanswered");
assert.equal((await env.root.sticky(ctx)).inbox.length, 0);
assert.equal(await a.abort(ctx), "already_placed");
assert.equal(await env.input(99999), undefined);
gate.open();
await a.wait(ctx);
});
test("reset while busy stales every earlier queued steer/followUp; later ones run in the fresh context", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
await gate.arrivals(1);
const b1 = await env.root.send({ content: "B1" }, ctx);
const b2 = await env.root.send({ content: "B2" }, ctx);
await env.root.reset(undefined, ctx);
const c = await env.root.send({ content: "C" }, ctx); // admitted after the reset
gate.open();
await a.wait(ctx);
await c.wait(ctx);
await env.root.waitForIdle(ctx);
assert.equal((await env.input(b1.id))?.reason, "stale");
assert.equal((await env.input(b2.id))?.reason, "stale");
assert.equal((await env.input(c.id))?.status, "done");
const { entries } = await env.root.commit((tx) => tx.context(1), ctx);
assert.equal(kinds(entries), "reset* user system assistant"); // C in the fresh context, with a fresh system baseline
});
test("handoff via tool control ends the turn with a head; a queued followUp starts in the fresh context", async () => {
const gate = new Gate();
const hand = tool("hand", { gate, result: { control: { handoff: "fresh" } } });
const env = await open({ tools: [hand] });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "tool:hand" }, ctx);
await gate.arrivals(1);
const f = await env.root.send({ content: "F" }, ctx);
gate.open();
await a.wait(ctx);
await f.wait(ctx);
await env.root.waitForIdle(ctx);
const post = (await env.tasks()).find((t) => t.kind === "pi.post_tools")!;
assert.equal((post.outcome as unknown as { result: { ended: string } }).result.ended, "handoff");
assert.equal((await env.input(a.id))?.status, "done");
assert.equal((await env.input(a.id))?.answer, (await env.entries()).find((e) => e.kind === "pi.assistant")!.id);
const { entries } = await env.root.commit((tx) => tx.context(1), ctx);
assert.equal(kinds(entries), "handoff* user system assistant");
});
test("terminate via tool control ends the turn without a head", async () => {
const t = tool("t", { result: { control: { terminate: true } } });
const env = await open({ tools: [t] });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "tool:t" }, ctx);
await a.wait(ctx);
await env.root.waitForIdle(ctx);
assert.equal(kinds(await env.entries()), "user system assistant tool_result"); // no continuation generation
assert.equal((await env.tasks()).filter((t) => t.kind === "pi.generation").length, 1);
});
test("addTools control extends selectedTools before the continuation", async () => {
const t = tool("t", { result: { control: { addTools: ["extra"] } } });
const extra = tool("extra");
const env = await open({ tools: [t, extra], root: { rewindable: { model, selectedTools: ["t"] } } });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "tool:t" }, ctx);
await a.wait(ctx);
assert.deepEqual((await env.root.rewindable(ctx)).selectedTools, ["t", "extra"]);
const sys = (await env.entries()).filter((e) => e.kind === "pi.system");
assert.equal(sys.length, 2); // loadout changed → a delta before the continuation, listing only what was added
assert.deepEqual(
(sys[1]!.model![0] as { toolsAdded: { name: string }[] }).toolsAdded.map((t) => t.name),
["extra"],
);
assert.equal((sys[1]!.model![0] as { toolsRemoved?: unknown[] }).toolsRemoved, undefined);
const { messages } = await env.root.commit((tx) => tx.context(1), ctx);
assert.deepEqual(
effectiveTools(messages).map((t) => t.name),
["t", "extra"],
);
});
test("retryable provider error: pi.usage recorded, retried, then succeeds", async () => {
let n = 0;
const models = fake({ respond: (m) => (n++ === 0 ? { error: "overloaded" } : echoScript(m)) });
const env = await open({
models,
root: { rewindable: { model }, sticky: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } } },
});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
const r = await a.wait(ctx);
assert.equal(r.status, "done");
assert.equal(kinds(await env.entries()), "user system usage assistant");
assert.equal(models.calls, 2);
assert.equal(((await env.entries())[2]!.data as { attempt: number }).attempt, 1);
});
test("retries exhausted → failed/retries_exhausted, inputs unanswered/failed, display-only assistant entry", async () => {
const models = fake({ respond: () => ({ error: "overloaded" }) });
const env = await open({
models,
root: { rewindable: { model }, sticky: { retry: { enabled: true, maxRetries: 1, baseDelayMs: 1 } } },
});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
const r = await a.wait(ctx);
assert.equal(r.status, "unanswered");
assert.equal(r.reason, "failed");
const gen = (await env.tasks()).find((t) => t.kind === "pi.generation")!;
assert.equal((gen.outcome as unknown as { failure: { reason: string } }).failure.reason, "retries_exhausted");
assert.equal(models.calls, 2); // initial + 1 retry
assert.equal(kinds(await env.entries()), "user system usage assistant");
const { messages } = await env.root.commit((tx) => tx.context(1), ctx);
assert.equal(messages.filter((m) => m.role === "assistant").length, 0); // display-only: never in context
const shown = (await env.entries()).find((e) => e.kind === "pi.assistant")!;
assert.equal(shown.model, undefined);
assert.equal((shown.data as { reason: string }).reason, "error");
});
test("retry disabled → failed/provider immediately", async () => {
const models = fake({ respond: () => ({ error: "boom" }) });
const env = await open({
models,
root: { rewindable: { model }, sticky: { retry: { enabled: false, maxRetries: 0, baseDelayMs: 1 } } },
});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
await a.wait(ctx);
const gen = (await env.tasks()).find((t) => t.kind === "pi.generation")!;
assert.equal((gen.outcome as unknown as { failure: { reason: string } }).failure.reason, "provider");
assert.equal(models.calls, 1);
});
test("no model → failed/no_model, no provider call", async () => {
const models = fake({ respond: echoScript });
const env = await open({ models, root: { rewindable: {} } });
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
await a.wait(ctx);
const gen = (await env.tasks()).find((t) => t.kind === "pi.generation")!;
assert.equal((gen.outcome as unknown as { failure: { reason: string } }).failure.reason, "no_model");
assert.equal(models.calls, 0);
});
test("yield continuation: onYield continues with a pi.user entry and the same input group", async () => {
let yielded = 0;
const env = await open({
hooks: { generation: { onYield: () => (yielded++ === 0 ? { continue: "keep going" } : undefined) } },
});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
const r = await a.wait(ctx);
await env.root.waitForIdle(ctx);
assert.equal(kinds(await env.entries()), "user system assistant user assistant");
const cont = (await env.entries())[3]!;
assert.deepEqual(cont.data, { continuation: true, from: (await env.entries())[2]!.id });
const gens = (await env.tasks()).filter((t) => t.kind === "pi.generation");
assert.deepEqual(
gens.map((g) => (g.input as { inputs: number[] }).inputs),
[[a.id], [a.id]],
);
assert.equal(r.answer, (await env.entries())[4]!.id); // answered by the second generation
});
test("conversation abort mid-stream: task aborted with partial, inputs unanswered/aborted, steer withdrawn, write kept", async () => {
const gate = new Gate();
const env = await open({
models: fake({
respond: (messages, call) => (call === 0 ? { text: "partial ".repeat(100) } : echoScript(messages)),
gate,
tokenDelayMs: 5,
}),
});
onTestFinished(() => env.close());
const a = await env.root.send({ content: "A" }, ctx);
await gate.arrivals(1);
const s = await env.root.send({ content: "S", whenBusy: "steer" }, ctx);
const w = await env.root.write({ kind: "pi.notice", model: [{ role: "user", content: "n", timestamp: 0 }] }, ctx);
gate.open();
for (let attempt = 0; attempt < 100; attempt++) {
if (
(await env.root.sticky(ctx)).turn.message?.content.some((part) => part.type === "text" && part.text.length > 0)
)
break;
await sleep(5);
}
assert.ok((await env.root.sticky(ctx)).turn.message, "the abort must happen after streamed content");
await env.root.abort(ctx);
assert.equal((await env.input(a.id))?.reason, "aborted");
assert.equal((await env.input(s.id))?.reason, "aborted");
assert.equal((await env.input(w))?.status, "queued"); // writes survive conversation abort
const gen = (await env.tasks()).find((t) => t.kind === "pi.generation")!;
assert.equal(gen.outcome?.status, "aborted");
const partial = (await env.entries()).find(
(entry) => entry.kind === "pi.assistant" && (entry.data as { reason?: string } | undefined)?.reason === "aborted",
)!;
assert.equal(partial.model, undefined);
assert.ok((partial.data as { display: { content: unknown[] } }).display.content.length > 0);
assert.deepEqual(
(await env.root.sticky(ctx)).inbox.map((q) => q.mode),
["write"],
);
// next idle send places the surviving write first
const b = await env.root.send({ content: "B" }, ctx);
await b.wait(ctx);
assert.equal(kinds(await env.entries()), "user system assistant notice user assistant");
});
test("overflow: collapse task + replacement generation with the same inputs", async () => {
const models = fake({
respond: (m) =>
m.some((x) => x.role === "user" && String((x as { content: unknown }).content).startsWith("Summarize"))
? { text: "summary text" }
: echoScript(m),
});
const modelObj = models.resolve({ provider: "", modelId: "" })!;
const env = await open({ models, root: { rewindable: { model, keepRecent: 100 } } });
onTestFinished(() => env.close());
// three turns of history, then shrink the window so the next request overflows
for (const t of ["one", "two", "three"]) await (await env.root.send({ content: t }, ctx)).wait(ctx);
(modelObj as { contextWindow: number }).contextWindow = 200;
(modelObj as { maxTokens: number }).maxTokens = 10;
const four = await env.root.send({ content: "four" }, ctx);
const r = await four.wait(ctx);
await env.root.waitForIdle(ctx);
const tasks = await env.tasks();
const collapse = tasks.find((t) => t.kind === "pi.collapse")!;
assert.ok(collapse, "collapse created");
assert.equal((collapse.input as { reason: string }).reason, "overflow");
const gens = tasks.filter((t) => t.kind === "pi.generation");
const overflowed = gens.find((g) => g.outcome?.status === "failed")!;
assert.equal((overflowed.outcome as unknown as { failure: { reason: string } }).failure.reason, "overflow");
const replacement = gens.find((g) => g.after.includes(collapse.id))!;
assert.deepEqual((replacement.input as { inputs: number[] }).inputs, [four.id]);
assert.equal(r.status, "done");
assert.ok((await env.entries()).some((e) => e.kind === "pi.summary"));
});

View file

@ -0,0 +1,21 @@
import type { Context } from "@earendil-works/chord";
import type { Runtime, TaskTx } from "../../../src/harness/pico3/types.ts";
declare const runtime: Runtime;
declare const context: Context;
runtime.commit((tx, _current, lineContext) => {
const taskTx: TaskTx = tx;
const propagated: Context = lineContext;
void taskTx;
void propagated;
// @ts-expect-error ordinary task transactions do not expose core entry admission
tx.appendEntry(1, { kind: "forged" });
// @ts-expect-error ordinary task transactions do not expose core inbox boundaries
tx.boundary(1, "final", undefined);
// @ts-expect-error ordinary task transactions may read config but cannot write it
tx.config(1).set("profile", "forged");
// @ts-expect-error ordinary task transactions do not expose raw core documents
tx.sticky(1);
return null;
}, context);

View file

@ -0,0 +1,299 @@
import assert from "node:assert/strict";
import { withAbortSignal } from "@earendil-works/chord/context";
import { Type } from "typebox";
import { onTestFinished, test } from "vitest";
import { applyEnvelope, type ConversationView, kinds } from "../../../src/harness/pico3/harness.ts";
import type { ToolDeclaration } from "../../../src/harness/pico3/types.ts";
import {
collectWatch,
contentOf,
ctx,
echoScript,
fake,
fakeHost,
Gate,
open,
sleep,
untilTerminal,
} from "./helpers.ts";
/** What a UI reads from the view. This is the whole rendering model. */
function uiModel(v: ConversationView) {
const turn = v.turn;
return {
transcript: v.entries.map((entry) => entry.kind.replace("pi.", "")),
streaming: turn?.message ? (turn.message.content[0] as { text?: string } | undefined)?.text : undefined,
runningTools: (turn?.tools ?? []).map((tool) => ({
name: tool.name,
output: tool.output,
continuedBy: tool.continuedBy,
})),
background: Object.entries(v.tasks)
.filter(([, task]) => task.background)
.map(([id, task]) => ({ id: Number(id), kind: task.kind, slot: task.status })),
working: turn !== undefined,
};
}
const turnOf = (v: ConversationView) => v;
test("turn view: streaming message → tools[] atomically with the assistant entry → cleared at final answer", async () => {
const gate = new Gate();
const toolGate = new Gate();
const t: ToolDeclaration = {
name: "x",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute(_a, api, c) {
api.stream("partial ");
await toolGate.wait(c);
api.stream("done");
return {};
},
};
const env = await open({ tools: [t], models: fake({ respond: echoScript, gate, tokenDelayMs: 2 }) });
onTestFinished(() => env.close());
const { view, envelopes, stop } = await collectWatch(env.root);
let v = view;
const fold = () => {
for (const d of envelopes.splice(0)) v = applyEnvelope(v, d);
return uiModel(turnOf(v));
};
const a = await env.root.send({ content: "tool:x" }, ctx);
await gate.arrivals(1);
gate.open();
await sleep(30); // a few frames of "Let me..." — echoScript's tool response has no text, so streaming shows toolcall only
await toolGate.arrivals(1);
const mid = fold();
assert.deepEqual(mid.transcript, ["user", "system", "assistant"]); // assistant entry landed…
assert.deepEqual(mid.runningTools, [{ name: "x", output: "partial ", continuedBy: undefined }]); // …with its tools, in the same envelope
assert.equal(mid.streaming, undefined);
assert.equal(mid.working, true);
// the envelope that appended the assistant entry also created turn.tools: no frame where one exists without the other
toolGate.open();
await a.wait(ctx);
await env.root.waitForIdle(ctx);
const end = fold();
assert.deepEqual(end.transcript, ["user", "system", "assistant", "tool_result", "assistant"]);
assert.deepEqual(end.runningTools, []);
assert.equal(end.streaming, undefined);
assert.equal(end.working, false);
stop();
});
test("streaming assistant text is visible in turn.message before the entry lands, and the fold equals a fresh snapshot", async () => {
const gate = new Gate();
const words = Array.from({ length: 40 }, (_, i) => `w${i}`).join(" ");
const env = await open({ models: fake({ respond: () => ({ text: words }), gate, tokenDelayMs: 10 }) });
onTestFinished(() => env.close());
const { view, envelopes, stop } = await collectWatch(env.root);
let v = view;
const fold = () => {
for (const d of envelopes.splice(0)) v = applyEnvelope(v, d);
return uiModel(turnOf(v));
};
const a = await env.root.send({ content: "hi" }, ctx);
await gate.arrivals(1);
gate.open();
await sleep(250);
const mid = fold();
assert.ok(
mid.streaming !== undefined && mid.streaming.length > 0 && mid.streaming.length < words.length,
`partial: ${JSON.stringify(mid.streaming)}`,
);
assert.deepEqual(mid.transcript, ["user", "system"]);
await a.wait(ctx);
const end = fold();
assert.equal(end.streaming, undefined);
assert.deepEqual(end.transcript, ["user", "system", "assistant"]);
const fresh = await collectWatch(env.root);
fresh.stop();
assert.deepEqual(v, fresh.view);
stop();
});
test("a tool backgrounds into a job: turn slot links continuedBy, the job streams into tasks[id], the turn ends, the job lives on", async () => {
const host = fakeHost();
const bg: ToolDeclaration = {
name: "bg",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute(_a, api, c) {
const job = await api.task(
kinds.job,
{ command: "sleep", args: [], cwd: "/", notify: false, rerun: false },
{ background: true },
c,
);
await api.progress((s) => {
s.continuedBy = job.id;
s.progress = "backgrounded";
}, c);
return {
content: [{ type: "text", text: `continues in background as task ${job.id}` }],
details: { jobId: job.id },
};
},
};
const env = await open({ tools: [bg], processHost: host, models: fake({ respond: echoScript }) });
onTestFinished(() => env.close());
const { view, envelopes, stop } = await collectWatch(env.root);
let v = view;
const fold = () => {
for (const d of envelopes.splice(0)) v = applyEnvelope(v, d);
return uiModel(turnOf(v));
};
const a = await env.root.send({ content: "tool:bg" }, ctx);
await a.wait(ctx);
await env.root.waitForIdle(ctx);
const jobId = (await env.tasks()).find((t) => t.kind === "pi.job")!.id;
const key = `${jobId}:1`;
// turn is over; the job is still running and visible under background with its own slot
for (let i = 0; i < 50 && !host.procs.has(key); i++) await sleep(5);
const m1 = fold();
assert.deepEqual(m1.runningTools, []); // turn cleared
assert.equal(m1.working, false);
assert.deepEqual(
m1.background.map((b) => b.kind),
["pi.job"],
);
// the tool result entry carries the link durably (turn slot is gone, but the result says where it went)
const tr = (await env.entries()).find((e) => e.kind === "pi.tool_result")!;
assert.equal((tr.data as { details: { jobId: number } }).details.jobId, jobId);
// job output streams into tasks[jobId]
host.procs.get(key)!.status = {
status: "running",
stdout: "job says hi",
stderr: "",
droppedStdout: 0,
droppedStderr: 0,
};
await sleep(150);
const m2 = fold();
assert.equal((m2.background[0]!.slot as { stdout: string }).stdout, "job says hi");
host.exit(key, 0);
await untilTerminal(env, jobId);
await sleep(20);
const m3 = fold();
assert.deepEqual(m3.background, []);
assert.deepEqual(v.tasks, {});
stop();
});
/** Start a job; watch it under this tool until `timeoutMs`; then let it continue in the background. */
function jobTool(timeoutMs: number): ToolDeclaration {
return {
name: "run",
description: "",
parameters: Type.Object({ v: Type.String() }),
async execute(_a, api, ctx) {
const job = await api.task(
kinds.job,
{ command: "x", args: [], cwd: "/", notify: false, rerun: false },
{ background: true },
ctx,
);
await api.progress((s) => {
s.continuedBy = job.id;
}, ctx); // UI renders tasks[job.id] under this tool from here on
const budget = withAbortSignal(AbortSignal.timeout(timeoutMs), ctx);
try {
const done = await api.waitForTask(job, budget); // the ref carries the kind: typed outcome, nothing else passed
if (done.outcome?.status !== "completed")
return { content: [{ type: "text", text: `job ${done.outcome?.status}` }], isError: true };
return {
content: [{ type: "text", text: done.outcome.result.stdout }],
isError: done.outcome.result.exitCode !== 0,
details: { jobId: job.id, finished: true },
};
} catch (error) {
if (ctx.abortSignal?.aborted) throw error; // a real abort, not our budget
// The model gets what the job produced so far, plus where to find the rest.
const soFar = (await api.slot(job, ctx))?.stdout ?? "";
return {
content: [{ type: "text", text: `${soFar}\n\n[still running in the background as task ${job.id}]` }],
details: { jobId: job.id, finished: false },
};
}
},
};
}
test("job under a tool: finishes within the budget → result is the job's output", async () => {
const host = fakeHost();
const env = await open({ tools: [jobTool(2000)], processHost: host, models: fake({ respond: echoScript }) });
onTestFinished(() => env.close());
const { view, envelopes, stop } = await collectWatch(env.root);
let v = view;
const fold = () => {
for (const d of envelopes.splice(0)) v = applyEnvelope(v, d);
return uiModel(turnOf(v));
};
const a = await env.root.send({ content: "tool:run" }, ctx);
let key = "";
for (let i = 0; i < 100 && !key; i++) {
await sleep(5);
key = [...host.procs.keys()][0] ?? "";
}
host.procs.get(key)!.status = {
status: "running",
stdout: "halfway",
stderr: "",
droppedStdout: 0,
droppedStderr: 0,
};
await sleep(150);
const mid = fold();
// UI: the running tool links to the job, whose live stdout is under background
assert.equal(mid.runningTools[0]!.continuedBy, mid.background[0]!.id);
assert.equal((mid.background[0]!.slot as { stdout: string }).stdout, "halfway");
host.exit(key, 0);
await a.wait(ctx);
await env.root.waitForIdle(ctx);
const tr = (await env.entries()).find((e) => e.kind === "pi.tool_result")!;
assert.equal(contentOf(tr), JSON.stringify([{ type: "text", text: `out ${key}` }]));
assert.equal((tr.data as { details: { finished: boolean } }).details.finished, true);
const end = fold();
assert.deepEqual(end.background, []);
stop();
});
test("job under a tool: budget expires → tool returns, job continues in background, turn ends", async () => {
const host = fakeHost();
const env = await open({ tools: [jobTool(300)], processHost: host, models: fake({ respond: echoScript }) });
onTestFinished(() => env.close());
const { view, envelopes, stop } = await collectWatch(env.root);
let v = view;
const fold = () => {
for (const d of envelopes.splice(0)) v = applyEnvelope(v, d);
return uiModel(turnOf(v));
};
const a = await env.root.send({ content: "tool:run" }, ctx);
let key = "";
for (let i = 0; i < 100 && !key; i++) {
await sleep(5);
key = [...host.procs.keys()][0] ?? "";
}
host.procs.get(key)!.status = {
status: "running",
stdout: "first 20 lines",
stderr: "",
droppedStdout: 0,
droppedStderr: 0,
};
await a.wait(ctx);
await env.root.waitForIdle(ctx);
const tr = (await env.entries()).find((e) => e.kind === "pi.tool_result")!;
assert.equal((tr.data as { details: { finished: boolean } }).details.finished, false);
assert.match(contentOf(tr), /first 20 lines[\s\S]*still running/); // the model sees the partial output AND the note
const m = fold();
assert.deepEqual(m.runningTools, []); // turn over
assert.equal(m.working, false);
assert.equal(m.background.length, 1); // the job lives on
const jobId = m.background[0]!.id;
assert.equal((await env.tasks()).find((t) => t.id === jobId)!.status, "running"); // the tool's timeout did not abort it
host.exit(`${jobId}:1`, 0);
await untilTerminal(env, jobId);
stop();
});

View file

@ -0,0 +1,85 @@
import assert from "node:assert/strict";
import { withAbortSignal } from "@earendil-works/chord/context";
import { onTestFinished, test } from "vitest";
import { kinds } from "../../../src/harness/pico3/harness.ts";
import { defineTask } from "../../../src/harness/pico3/types.ts";
import { ctx, echoScript, fake, Gate, open } from "./helpers.ts";
test("waitForTask/waitForInput/waitForIdle: registration is atomic with the state read; completion racing the registration never strands a waiter", async () => {
const env = await open({});
onTestFinished(() => env.close());
const quick = defineTask<null, { phase: "x" }, null, null, null>({
name: "quick",
async initial() {
return { next: { phase: "x" } };
},
phases: {
async x() {
return { done: () => ({ status: "completed", result: null }) };
},
},
async abort() {
return () => null;
},
});
const env2 = await open({ taskKinds: [quick] });
onTestFinished(() => env2.close());
// 200 iterations: create and immediately wait; the task may terminalize before, during, or after the waiter enters the line.
for (let i = 0; i < 200; i++) {
const ref = await env2.root.commit(
(tx) => tx.createTask(quick, null, { conversationId: 1, background: true }),
ctx,
);
const t = await env2.h.waitForTask(ref.id, ctx);
assert.equal(t.status, "terminal");
}
for (let i = 0; i < 50; i++) {
const h = await env.root.send({ content: `m${i}` }, ctx);
const [r] = await Promise.all([h.wait(ctx), env.root.waitForIdle(ctx), env.h.waitForIdle(ctx)]);
assert.equal(r.status, "done");
}
// an already-terminal task returns immediately
const first = (await env2.tasks())[0]!;
assert.equal((await env2.h.waitForTask(first.id, ctx)).status, "terminal");
});
test("waiters: an aborted context rejects the waiter only; the durable work continues; an already-aborted context rejects immediately", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const h = await env.root.send({ content: "A" }, ctx);
const ac = new AbortController();
const cctx = withAbortSignal(ac.signal, ctx);
const p = h.wait(cctx);
const q = env.root.waitForIdle(cctx);
const gen = (await env.tasks()).find((t) => t.kind === "pi.generation")!;
const r = env.h.waitForTask(gen.id, cctx);
await gate.arrivals(1);
ac.abort(new Error("gone"));
await assert.rejects(p, /gone/);
await assert.rejects(q, /gone/);
await assert.rejects(r, /gone/);
const dead = new AbortController();
dead.abort(new Error("pre"));
await assert.rejects(h.wait(withAbortSignal(dead.signal, ctx)), /pre/);
gate.open();
assert.equal((await h.wait(ctx)).status, "done"); // the turn finished regardless
});
test("Harness.waitForIdle does not resolve while a drain is about to dispatch a just-created task", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const h = await env.root.send({ content: "A" }, ctx);
let idleResolved = false;
const idle = env.h.waitForIdle(ctx).then(() => {
idleResolved = true;
});
await gate.arrivals(1);
assert.equal(idleResolved, false);
gate.open();
await h.wait(ctx);
await idle;
assert.equal((await env.tasks()).filter((t) => t.status !== "terminal").length, 0);
void kinds;
});

View file

@ -0,0 +1,192 @@
import assert from "node:assert/strict";
import { onTestFinished, test } from "vitest";
import {
applyEnvelope,
type ConversationView,
type Envelope,
WATCH_CAPACITY,
} from "../../../src/harness/pico3/harness.ts";
import { ctx, echoScript, fake, Gate, open } from "./helpers.ts";
const shape = (v: ConversationView) =>
v.entries.map((e) => `${e.id}:${e.kind.replace("pi.", "")}${e.head !== undefined ? `>${e.head}` : ""}`).join(" ");
test("watch: envelopes have contiguous revisions; one per visible commit; folded view equals a fresh capture", async () => {
const env = await open({});
onTestFinished(() => env.close());
const w = await env.root.watch(ctx);
const revisions: number[] = [];
let folded = w.view;
w.start((envelope) => {
revisions.push(envelope.revision);
folded = applyEnvelope(folded, envelope);
});
for (const t of ["one", "two"]) await (await env.root.send({ content: t }, ctx)).wait(ctx);
await env.root.write({ kind: "note" }, ctx);
for (let index = 1; index < revisions.length; index++) assert.equal(revisions[index], revisions[index - 1]! + 1);
const fresh = await env.root.watch(ctx);
assert.equal(shape(folded), shape(fresh.view));
assert.deepEqual(folded, fresh.view);
fresh.stop();
w.stop();
w.stop(); // idempotent
const delivered = revisions.length;
await env.root.write({ kind: "note" }, ctx);
assert.equal(revisions.length, delivered); // no callbacks after stop
});
test("failed document commits do not break queued-input abort or later task-only watch updates", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
const active = await env.root.send({ content: "active" }, ctx);
await gate.arrivals(1);
const queued = await env.root.send({ content: "queued" }, ctx);
const generation = (await env.tasks()).find((task) => task.kind === "pi.generation" && task.status !== "terminal")!;
const watch = await env.root.watch(ctx);
watch.start(() => {});
await assert.rejects(
env.root.commit((tx) => {
tx.config(env.root.id).set("profile", "rolled-back");
throw new Error("rollback");
}, ctx),
/rollback/,
);
assert.equal(await queued.abort(ctx), "aborted");
assert.equal((await queued.result(ctx))?.reason, "aborted");
assert.equal(await env.h.markTask(generation.id, ctx), "marked");
assert.equal(watch.closed, false);
await env.h.abortTask(generation.id, ctx);
await active.wait(ctx);
watch.stop();
});
test("watch: a listener throw closes only that watch, is reported, and never fails the writer; a second watch keeps going", async () => {
const reports: unknown[] = [];
const env = await open({ onReport: (e) => reports.push(e) });
onTestFinished(() => env.close());
const bad = await env.root.watch(ctx);
const good = await env.root.watch(ctx);
let goodCount = 0;
bad.start(() => {
throw new Error("bad listener");
});
good.start(() => {
goodCount++;
});
const id = await env.root.write({ kind: "note" }, ctx); // resolves normally
assert.ok(id > 0);
assert.equal(bad.closed, true);
assert.equal(good.closed, false);
assert.equal(goodCount, 1);
assert.match(String(reports[0]), /bad listener/);
});
test("watch: bounded capacity before start(); overflow closes the watch and reports; the client opens a fresh watch", async () => {
const reports: unknown[] = [];
const env = await open({ onReport: (e) => reports.push(e) });
onTestFinished(() => env.close());
const w = await env.root.watch(ctx);
for (let i = 0; i <= WATCH_CAPACITY; i++) await env.root.write({ kind: "note" }, ctx);
assert.equal(w.closed, true);
assert.match(String(reports[0]), /capacity 256 exceeded/);
const fresh = await env.root.watch(ctx);
assert.equal(fresh.view.entries.filter((e) => e.kind === "note").length, WATCH_CAPACITY + 1);
fresh.stop();
});
test("watch capture: fork-inherited head target, older head inside the range, and capture equals incremental fold", async () => {
const gate = new Gate();
const env = await open({ models: fake({ respond: echoScript, gate }) });
onTestFinished(() => env.close());
gate.open();
await (await env.root.send({ content: "one" }, ctx)).wait(ctx);
await (await env.root.send({ content: "two" }, ctx)).wait(ctx);
await env.root.reset("carry on", ctx); // head H1 in the parent
await (await env.root.send({ content: "three" }, ctx)).wait(ctx);
const parentEntries = await env.entries();
const h1 = parentEntries.find((e) => e.kind === "pi.handoff")!;
const tip = parentEntries[parentEntries.length - 1]!;
// child forks at the parent's tip; its own newest head points INTO the parent (at H1's position, i.e. an inherited entry)
const child = await env.root.fork(tip.id, {}, ctx);
const cw = await child.watch(ctx);
let folded = cw.view;
cw.start((envelope) => {
folded = applyEnvelope(folded, envelope);
});
// add a passive entry, a generated exchange, then a head that points at the inherited handoff
await child.write({ kind: "note" }, ctx);
await (await child.send({ content: "four" }, ctx)).wait(ctx);
await child.reset(undefined, ctx); // head H2 = self, later than H1
const fresh = await child.watch(ctx);
assert.equal(shape(fresh.view), shape(folded), "incremental fold equals a fresh capture");
// the range starts at H2.head == H2 itself: exactly the reset entry
const heads = fresh.view.entries.filter((e) => e.head !== undefined);
assert.equal(heads.length, 1);
fresh.stop();
cw.stop();
// Inherited case: a second child forked at the parent's tip whose newest head is the parent's H1 (inherited) — capture includes the inherited target and everything after it.
const child2 = await env.root.fork(tip.id, {}, ctx);
const cw2 = await child2.watch(ctx);
const ids = cw2.view.entries.map((e) => e.id);
assert.ok(ids.includes(h1.id), "inherited head entry retained at its position");
assert.deepEqual(
ids,
parentEntries.filter((e) => e.id >= h1.id).map((e) => e.id),
);
// an older head INSIDE the range: the view keeps it (context derivation would not)
let folded2 = cw2.view;
cw2.start((envelope) => {
folded2 = applyEnvelope(folded2, envelope);
});
await child2.reset("again", ctx); // H2 in child2; range = [H2]
await (await child2.send({ content: "five" }, ctx)).wait(ctx);
const fresh2 = await child2.watch(ctx);
assert.equal(shape(fresh2.view), shape(folded2));
fresh2.stop();
cw2.stop();
});
test("applyEnvelope folds head truncation ops while preserving display-only and older in-range head entries", () => {
const base: ConversationView = {
conversation: { id: 1 },
entries: [
{ id: 1, conversationId: 1, kind: "pi.user" },
{ id: 2, conversationId: 1, kind: "pi.summary", head: 2 },
{ id: 3, conversationId: 1, kind: "pi.assistant" },
],
config: {},
inbox: [],
tasks: {},
plugins: {},
};
const display = { id: 4, conversationId: 1, kind: "pi.assistant", data: { reason: "aborted" } };
const handoff = { id: 5, conversationId: 1, kind: "pi.handoff", head: 3 };
const first: Envelope = {
revision: 9,
ops: [
["p", ["entries"], 0, 2, []],
["p", ["entries"], 1, 0, [display, handoff]],
],
events: [],
};
const view = applyEnvelope(base, first);
assert.deepEqual(
view.entries.map((entry) => entry.id),
[3, 4, 5],
);
const summary = { id: 6, conversationId: 1, kind: "pi.summary", head: 4 };
const second: Envelope = {
revision: 10,
ops: [
["p", ["entries"], 0, 1, []],
["p", ["entries"], 2, 0, [summary]],
],
events: [],
};
const next = applyEnvelope(view, second);
assert.deepEqual(
next.entries.map((entry) => entry.id),
[4, 5, 6],
);
});