mirror of
https://github.com/badlogic/pi-mono.git
synced 2026-11-05 23:37:25 +00:00
7b597d1481
A codemode tool whose scripts call the conversation's tools as nested calls and survive a restart. The script runs deterministically (pi-codemode start()); each result is journaled before it is delivered. A rerun starts the script again from the stored start data, delivers the journaled results in order (nested calls reattach by key c<id>), checks every recorded call count, holds calls the journal lacks until replay ends and checks them against earlier attempts' keys, then continues live. A call launches only once the count of the drain that made it is committed, so a rerun can prove which calls are new. Divergence, a changed module version or sandbox fingerprint, and a passed deadline end the call with a stored error; a decided outcome is returned without a VM. store()/load() keep values per conversation, rewindable and forked as of the fork's entry. pi-codemode: CodemodeRun.finished, set when the outcome is decided.
722 lines
29 KiB
TypeScript
722 lines
29 KiB
TypeScript
/**
|
|
* Durable code mode: a tool that runs model-written JavaScript in a QuickJS sandbox, whose script survives a restart.
|
|
*
|
|
* Problem: a script is in-memory VM state. A restart while
|
|
*
|
|
* const results = await Promise.all(paths.map((path) => tools.read({ path })));
|
|
*
|
|
* waits for the last of 200 reads would lose the script and every read it made.
|
|
*
|
|
* Solution: deterministic replay (`CodemodeSandboxBase.start()` in pi-codemode). The VM only runs when the host hands
|
|
* it a result; given the same start data, clock, seed, and results in the same order, it makes the same calls with
|
|
* the same arguments. The tool is replay-safe: after a restart, `execute()` runs again, starts the script again from
|
|
* the stored start data, and hands it the results it got before, in the order it got them, from a journal written
|
|
* before each delivery. Nested tool calls reattach to their tasks by key (`c<VM call id>`), so they never run twice
|
|
* unless their own replay policy reruns them; their results come from `NestedCallDoc`. Then the script continues live.
|
|
*
|
|
* Documents, scoped to the code mode call's task and gone when it settles:
|
|
* - `pi.codemode.run`: the start data, written once before the VM starts, and the outcome, written once when the call
|
|
* decides it (with the store writes, in one transaction), so a rerun after that returns it without a VM.
|
|
* - `pi.codemode.journal`: one entry per delivered result, in delivery order: the VM call ID, the clock value, the
|
|
* calls the VM had issued after processing it (once acknowledged), and a global's outcome, which nothing else stores.
|
|
*
|
|
* Each result's acknowledged call count is recorded with the next journal commit, or at once when the VM has processed
|
|
* everything delivered: a script waiting on a long call is where a restart usually finds it. A sequential script
|
|
* therefore costs one journal commit and one count commit per result, on top of its nested calls' own; that is the
|
|
* price of checking the most common crash point.
|
|
*
|
|
* A call is launched only once the count of calls issued in the drain that made it is committed. So every key an earlier
|
|
* attempt admitted is at most its last recorded count; a replay checks every recorded count, so when it releases its
|
|
* calls it has made every call an earlier attempt could have admitted, and later calls are new.
|
|
*
|
|
* A rerun refuses to replay (no new nested calls; nested calls admitted earlier may still resume on their own and are
|
|
* aborted by the call's cleanup) when this module's version or the VM's fingerprint changed, when the deadline passed,
|
|
* or when the script diverges: it makes another call, or calls in another order, than the journal says. Each ends the
|
|
* call with an error result.
|
|
*/
|
|
import { type Context, copyJson, type JsonValue } from "@earendil-works/chord";
|
|
import type { ImageContent, TextContent } from "@earendil-works/pi-ai";
|
|
import {
|
|
type CodemodeHostCall,
|
|
type CodemodeOutcome,
|
|
type CodemodeResult,
|
|
type CodemodeRun,
|
|
type CodemodeSandboxBase,
|
|
toCodemodeIdentifier,
|
|
} from "@earendil-works/pi-codemode/portable";
|
|
import { parseCodemodeSource } from "@earendil-works/pi-codemode/source";
|
|
import { type Static, Type } from "typebox";
|
|
import { defineDoc } from "../documents.ts";
|
|
import { errorMessage } from "../errors.ts";
|
|
import { defineExtension, defineTool } from "../harness/define.ts";
|
|
import { NestedCallDoc, type ToolTaskInput } from "../harness/tool.ts";
|
|
import type {
|
|
Extension,
|
|
NestedToolExecutionResult,
|
|
ToolExecutionApi,
|
|
ToolExecutionResult,
|
|
ToolRegistration,
|
|
} from "../harness/types.ts";
|
|
import type { JsonObject } from "../types.ts";
|
|
|
|
/**
|
|
* Version of everything in this module that turns stored results into what a script sees: result mapping, globals,
|
|
* start data. Bump it when that changes, so a call started before an upgrade is not replayed by different code.
|
|
*/
|
|
export const CODEMODE_MODULE_VERSION = 1;
|
|
|
|
export const CODEMODE_TOOL_NAME = "codemode";
|
|
|
|
/** One tool as the script sees it: `tools.<jsName>`. */
|
|
type DeclaredTool = { name: string; jsName: string; description: string };
|
|
|
|
/** What a call starts with; a rerun starts the VM from exactly this. */
|
|
type Start = {
|
|
version: number;
|
|
startedAt: number;
|
|
seed: number;
|
|
/** The VM's fingerprint, recorded by the first run before any call is admitted. */
|
|
fingerprint?: string;
|
|
/** `startedAt + timeout_ms` when the script set one. */
|
|
deadline?: number;
|
|
tools: DeclaredTool[];
|
|
globals: { name: string; spread: boolean }[];
|
|
store: Record<string, JsonValue>;
|
|
};
|
|
|
|
type RunState = {
|
|
start?: Start;
|
|
/** The outcome, decided once. */
|
|
result?: JsonObject;
|
|
};
|
|
|
|
export const CodemodeRunDoc = defineDoc<RunState>({
|
|
kind: "pi.codemode.run",
|
|
version: 1,
|
|
scope: "task",
|
|
initial: () => ({}),
|
|
});
|
|
|
|
type JournalEntry = {
|
|
/** The VM's call ID. Its result is the entry's position + 1 (`seq`). */
|
|
id: number;
|
|
/** The VM clock while it processes this result. */
|
|
at: number;
|
|
/** Calls the VM had issued after processing this result; absent until acknowledged. */
|
|
issuedAfter?: number;
|
|
/** A global's call and outcome; a tool's result is its nested call's. */
|
|
global?: { name: string; args?: JsonValue; outcome: JsonObject };
|
|
};
|
|
|
|
type JournalState = {
|
|
entries: JournalEntry[];
|
|
/** Calls the VM had issued after the script's first run, before any result; absent until acknowledged. */
|
|
firstIssued?: number;
|
|
};
|
|
|
|
export const CodemodeJournalDoc = defineDoc<JournalState>({
|
|
kind: "pi.codemode.journal",
|
|
version: 1,
|
|
scope: "task",
|
|
initial: () => ({ entries: [] }),
|
|
});
|
|
|
|
/**
|
|
* What scripts keep with `store()` and read with `load()`, per conversation. Rewindable and copied as of the fork
|
|
* point, so a fork sees the values as they were when its entry was committed.
|
|
*/
|
|
export const CodemodeStoreDoc = defineDoc<{ values: Record<string, JsonValue> }>({
|
|
kind: "pi.codemode.store",
|
|
version: 1,
|
|
scope: "conversation",
|
|
history: "rewindable",
|
|
fork: "asOf",
|
|
initial: () => ({ values: {} }),
|
|
});
|
|
|
|
/** A function scripts call as a top-level global; its result is journaled. */
|
|
export type CodemodeGlobal = {
|
|
readonly name: string;
|
|
readonly spread?: boolean;
|
|
execute(args: unknown, api: ToolExecutionApi, context: Context): Promise<JsonValue | undefined>;
|
|
};
|
|
|
|
const codemodeSchema = Type.Object({ code: Type.String({ description: "Raw JavaScript source." }) });
|
|
|
|
export type CodemodeToolInput = Static<typeof codemodeSchema>;
|
|
|
|
export type CodemodeToolOptions = {
|
|
/** Runs the scripts: `CodemodeSandbox` on Node, `RemoteCodemodeSandbox` on Cloudflare. */
|
|
readonly sandbox: CodemodeSandboxBase;
|
|
/** @internal Host functions scripts call as globals. */
|
|
readonly globals?: readonly CodemodeGlobal[];
|
|
};
|
|
|
|
/** The code mode tool. Scripts call the conversation's callable tools as `tools.<name>(args)`. */
|
|
export function createCodemodeTool(options: CodemodeToolOptions): ToolRegistration<typeof codemodeSchema> {
|
|
return defineTool({
|
|
name: CODEMODE_TOOL_NAME,
|
|
description:
|
|
"Run JavaScript that calls other tools. The input is raw JavaScript, run as an async function body in a QuickJS sandbox: top-level `await` and `return` work. No Node, file system, network, or timers. `await tools.<name>({ ...args })` resolves to the tool's result and rejects with an Error on failure. `text(value)` and `console.log()` add output.",
|
|
parameters: codemodeSchema,
|
|
callers: ["model"],
|
|
replay: "safe",
|
|
execute: (args, api, context) => new CodemodeCall(options, api, context).execute(args.code),
|
|
});
|
|
}
|
|
|
|
/** The `codemode` extension: the code mode tool. */
|
|
export function createCodemodeExtension(options: CodemodeToolOptions): Extension {
|
|
return defineExtension({ name: "codemode", tools: [createCodemodeTool(options)] });
|
|
}
|
|
|
|
/** What a script gets for a nested call: success resolves with its value; every error rejects (design §4.5). */
|
|
function outcomeOf(name: string, result: NestedToolExecutionResult): CodemodeOutcome {
|
|
const value = result.structuredOutput;
|
|
if (!result.isError) return { ok: true, value };
|
|
const message =
|
|
typeof value === "string" && value !== ""
|
|
? value
|
|
: Array.isArray(value)
|
|
? value.flatMap((item) => (isText(item) ? [item.text] : [])).join("\n")
|
|
: "";
|
|
const diagnostics = result.diagnostics
|
|
.filter((diagnostic) => diagnostic.severity === "error")
|
|
.map((diagnostic) => diagnostic.message)
|
|
.join("\n");
|
|
return {
|
|
ok: false,
|
|
message: message || diagnostics || `Tool ${name} failed`,
|
|
...(value === undefined ? {} : { properties: { structuredOutput: value } }),
|
|
};
|
|
}
|
|
|
|
function isText(item: unknown): item is TextContent {
|
|
return typeof item === "object" && item !== null && (item as TextContent).type === "text";
|
|
}
|
|
|
|
/** The tool result for a script's result. */
|
|
function toolResult(result: CodemodeResult): ToolExecutionResult {
|
|
const output: (TextContent | ImageContent)[] = [];
|
|
for (const item of result.output) output.push(item.type === "image" ? item : { type: "text", text: item.text });
|
|
if (result.ok && result.value !== undefined) {
|
|
output.push({
|
|
type: "text",
|
|
text: typeof result.value === "string" ? result.value : JSON.stringify(result.value),
|
|
});
|
|
}
|
|
if (!result.ok) {
|
|
const { error } = result;
|
|
output.push({ type: "text", text: `Script error (${error.kind}): ${error.stack ?? error.message}` });
|
|
}
|
|
return { output, isError: !result.ok };
|
|
}
|
|
|
|
function errorResult(message: string): ToolExecutionResult {
|
|
return { output: [{ type: "text", text: message }], isError: true };
|
|
}
|
|
|
|
/** One execution of the code mode tool: a first run, or a rerun after a restart that replays the journal. */
|
|
class CodemodeCall {
|
|
private readonly options: CodemodeToolOptions;
|
|
private readonly api: ToolExecutionApi;
|
|
private readonly context: Context;
|
|
private readonly controller = new AbortController();
|
|
private run!: CodemodeRun;
|
|
private start!: Start;
|
|
/** The journal as this attempt found it; replay delivers exactly these. */
|
|
private replayed: JournalEntry[] = [];
|
|
/** Entries in the journal, replayed and live; the next delivery is `length + 1`. */
|
|
private journalLength = 0;
|
|
/** Replayed entries delivered so far. */
|
|
private nextReplayed = 0;
|
|
private replaying = false;
|
|
/** Whether an earlier attempt ran this call: its nested call keys may already exist. */
|
|
private rerun = false;
|
|
/** Whether live calls may launch; before, they are held (see `release()`). */
|
|
private released = false;
|
|
/** The count an earlier attempt recorded after the script's first run. */
|
|
private firstIssued: number | undefined;
|
|
/** The highest issued count known to be committed: calls up to it may launch. */
|
|
private durableIssued = 0;
|
|
/** Live calls waiting for their issued count to be committed. */
|
|
private waiting: CodemodeHostCall[] = [];
|
|
/** Journal index of each replayed call ID. */
|
|
private replayIndex = new Map<number, number>();
|
|
/** Calls the VM issued so far; its call IDs are 1, 2, ... */
|
|
private issued = 0;
|
|
/** Replay: results of journaled calls, ready to deliver in journal order. */
|
|
private readonly ready = new Map<number, CodemodeOutcome>();
|
|
/** Replay: calls the journal does not have, admitted once replay is done. */
|
|
private held: CodemodeHostCall[] = [];
|
|
/** Live: finished calls waiting for their journal commit. */
|
|
private queue: { id: number; at: number; outcome: CodemodeOutcome; global?: JournalEntry["global"] }[] = [];
|
|
/** Acknowledged counts to record, by result number (0: the script's first run). */
|
|
private readonly acks = new Map<number, number>();
|
|
private committing = false;
|
|
/** Set once the call decided an outcome other than the script's own (divergence, mismatch) or failed. */
|
|
private ended: ToolExecutionResult | undefined;
|
|
private failure: unknown;
|
|
/** Resolves once the VM's fingerprint is checked or recorded; nothing is delivered or admitted before. */
|
|
private gate!: Promise<void>;
|
|
private openGate!: () => void;
|
|
|
|
constructor(options: CodemodeToolOptions, api: ToolExecutionApi, context: Context) {
|
|
this.options = options;
|
|
this.api = api;
|
|
this.context = context;
|
|
this.gate = new Promise((resolve) => {
|
|
this.openGate = resolve;
|
|
});
|
|
}
|
|
|
|
async execute(source: string): Promise<ToolExecutionResult> {
|
|
const { code, options: sourceOptions } = parseCodemodeSource(source);
|
|
const { api, context } = this;
|
|
const agent = await api.agent(context);
|
|
const tools: DeclaredTool[] = agent.callable.map((tool) => ({
|
|
name: tool.name,
|
|
jsName: toCodemodeIdentifier(tool.name),
|
|
description: tool.description,
|
|
}));
|
|
const globals = (this.options.globals ?? []).map((global) => ({
|
|
name: global.name,
|
|
spread: global.spread === true,
|
|
}));
|
|
const now = Date.now();
|
|
const seed = crypto.getRandomValues(new Uint32Array(1))[0]!;
|
|
type Found =
|
|
| { done: ToolExecutionResult }
|
|
| { start: Start; journal: JournalEntry[]; firstIssued: number | undefined; rerun: boolean };
|
|
const found = await api.commit(async (tx): Promise<Found> => {
|
|
const run = await tx.doc(CodemodeRunDoc, api.taskId);
|
|
if (run.result !== undefined) return { done: copyJson(run.result as JsonValue) as ToolExecutionResult };
|
|
const rerun = run.start !== undefined;
|
|
if (run.start === undefined) {
|
|
const store = await tx.doc(CodemodeStoreDoc, api.conversationId);
|
|
const timeout = sourceOptions.timeoutMs;
|
|
run.start = {
|
|
version: CODEMODE_MODULE_VERSION,
|
|
startedAt: now,
|
|
seed,
|
|
...(timeout === undefined ? {} : { deadline: now + timeout }),
|
|
tools,
|
|
globals,
|
|
store: copyJson(store.values as JsonValue) as Record<string, JsonValue>,
|
|
};
|
|
}
|
|
const journal = await tx.doc(CodemodeJournalDoc, api.taskId);
|
|
return {
|
|
start: copyJson(run.start as unknown as JsonValue) as unknown as Start,
|
|
journal: copyJson(journal.entries as unknown as JsonValue) as unknown as JournalEntry[],
|
|
firstIssued: journal.firstIssued,
|
|
rerun,
|
|
};
|
|
}, context);
|
|
if ("done" in found) return found.done;
|
|
this.start = found.start;
|
|
this.replayed = found.journal;
|
|
this.replayIndex = new Map(found.journal.map((entry, index) => [entry.id, index]));
|
|
this.journalLength = found.journal.length;
|
|
this.replaying = found.journal.length > 0;
|
|
this.rerun = found.rerun;
|
|
this.firstIssued = found.firstIssued;
|
|
// A loop, not spread arguments: a journal can hold more entries than a call takes arguments.
|
|
this.durableIssued = found.journal.reduce(
|
|
(max, entry) => Math.max(max, entry.issuedAfter ?? 0),
|
|
found.firstIssued ?? 0,
|
|
);
|
|
|
|
if (this.start.version !== CODEMODE_MODULE_VERSION) {
|
|
return this.decide(
|
|
errorResult(
|
|
`The script cannot resume: it started under code mode version ${this.start.version}, this is ${CODEMODE_MODULE_VERSION}.`,
|
|
),
|
|
);
|
|
}
|
|
const remaining = this.start.deadline === undefined ? undefined : this.start.deadline - Date.now();
|
|
if (remaining !== undefined && remaining <= 0) {
|
|
return this.decide(errorResult(`Script timed out: its deadline passed while it was interrupted.`));
|
|
}
|
|
|
|
const onAbort = () => this.controller.abort(context.abortSignal?.reason);
|
|
context.abortSignal?.addEventListener("abort", onAbort, { once: true });
|
|
if (context.abortSignal?.aborted) onAbort();
|
|
try {
|
|
this.run = this.options.sandbox.start(code, {
|
|
startedAt: this.start.startedAt,
|
|
seed: this.start.seed,
|
|
tools: this.start.tools,
|
|
globals: this.start.globals,
|
|
store: this.start.store,
|
|
signal: this.controller.signal,
|
|
...(remaining === undefined ? {} : { timeoutMs: remaining }),
|
|
onStarted: (fingerprint) => void this.checkFingerprint(fingerprint),
|
|
onCall: (call) => this.onCall(call),
|
|
onIdle: (seq, issued) => this.onIdle(seq, issued),
|
|
});
|
|
const result = await this.run.result;
|
|
if (this.failure !== undefined) throw this.failure;
|
|
if (this.ended !== undefined) return this.ended;
|
|
context.abortSignal?.throwIfAborted();
|
|
// A replay that ends where the live run went on is a divergence (design §4.3).
|
|
const during = result.during;
|
|
// Within the replayed part, the first run included: a rerun without journal entries replays that much too.
|
|
if (this.rerun && during !== undefined && during.seq <= this.replayed.length) {
|
|
// The live run went on: it acknowledged that result, or it made a call that later results answer, which the
|
|
// replay never made.
|
|
const continued =
|
|
(during.seq === 0
|
|
? this.replayed.length > 0 || this.firstIssued !== undefined
|
|
: this.replayed[during.seq - 1]?.issuedAfter !== undefined) ||
|
|
this.replayed.slice(during.seq).some((entry) => entry.id > during.issued);
|
|
if (continued) {
|
|
return this.decide(
|
|
errorResult(
|
|
`The script took a different path when resumed after a restart: it ended after result ${during.seq}, where it went on before.`,
|
|
),
|
|
);
|
|
}
|
|
}
|
|
return this.complete(result);
|
|
} finally {
|
|
context.abortSignal?.removeEventListener("abort", onAbort);
|
|
}
|
|
}
|
|
|
|
/** Store an outcome the call decided, and end the VM if it runs; returns the outcome. */
|
|
private async decide(result: ToolExecutionResult): Promise<ToolExecutionResult> {
|
|
await this.api.commit(async (tx) => {
|
|
const run = await tx.doc(CodemodeRunDoc, this.api.taskId);
|
|
run.result ??= copyJson(result as JsonValue, { omitUndefinedProperties: true }) as JsonObject;
|
|
}, this.context);
|
|
return result;
|
|
}
|
|
|
|
/** End the call with `message`: store it as the outcome, then stop the VM with an error the script cannot catch. */
|
|
private diverge(message: string): void {
|
|
if (this.ended !== undefined || this.failure !== undefined) return;
|
|
const result = errorResult(`The script took a different path when resumed after a restart: ${message}`);
|
|
this.ended = result;
|
|
this.decide(result).then(
|
|
() => this.controller.abort(new Error(message)),
|
|
(error: unknown) => this.fail(error),
|
|
);
|
|
}
|
|
|
|
/** Whether nothing more may happen: the run has its outcome, or the call decided or failed. */
|
|
private get stopped(): boolean {
|
|
return this.ended !== undefined || this.failure !== undefined || this.run?.finished === true;
|
|
}
|
|
|
|
private fail(error: unknown): void {
|
|
if (this.failure !== undefined) return;
|
|
this.failure = error;
|
|
this.controller.abort(error);
|
|
}
|
|
|
|
/** The script's own outcome, stored with its store writes in one transaction. */
|
|
private async complete(result: CodemodeResult): Promise<ToolExecutionResult> {
|
|
const tool = toolResult(result);
|
|
const writes = result.ok ? result.storeWrites : undefined;
|
|
return this.api.commit(async (tx) => {
|
|
const run = await tx.doc(CodemodeRunDoc, this.api.taskId);
|
|
if (run.result !== undefined) return run.result as ToolExecutionResult;
|
|
run.result = copyJson(tool as JsonValue, { omitUndefinedProperties: true }) as JsonObject;
|
|
if (writes !== undefined && (Object.keys(writes.set).length > 0 || writes.delete.length > 0)) {
|
|
const store = await tx.doc(CodemodeStoreDoc, this.api.conversationId);
|
|
for (const key of writes.delete) delete store.values[key];
|
|
for (const [key, value] of Object.entries(writes.set)) store.values[key] = value as JsonValue;
|
|
}
|
|
return tool;
|
|
}, this.context);
|
|
}
|
|
|
|
private async checkFingerprint(fingerprint: string): Promise<void> {
|
|
const recorded = this.start.fingerprint;
|
|
if (recorded !== undefined) {
|
|
if (recorded !== fingerprint) {
|
|
this.ended = errorResult(
|
|
`The script cannot resume: the sandbox changed since it started (${recorded} -> ${fingerprint}).`,
|
|
);
|
|
await this.decide(this.ended).catch((error: unknown) => this.fail(error));
|
|
this.controller.abort(new Error("Sandbox fingerprint changed"));
|
|
return;
|
|
}
|
|
} else {
|
|
try {
|
|
await this.api.commit(async (tx) => {
|
|
const run = await tx.doc(CodemodeRunDoc, this.api.taskId);
|
|
if (run.start !== undefined) run.start.fingerprint = fingerprint;
|
|
}, this.context);
|
|
} catch (error) {
|
|
this.fail(error);
|
|
return;
|
|
}
|
|
}
|
|
this.openGate();
|
|
// Nothing was delivered before the gate: start now. A rerun without a journal releases its calls at the
|
|
// script's first acknowledgment instead.
|
|
if (this.replaying) this.deliverReplayed();
|
|
else if (!this.rerun) void this.release();
|
|
}
|
|
|
|
private onCall(call: CodemodeHostCall): void {
|
|
if (this.stopped) return;
|
|
this.issued = Math.max(this.issued, call.id);
|
|
const index = this.replaying ? (this.replayIndex.get(call.id) ?? -1) : -1;
|
|
if (index === -1) {
|
|
if (this.released) {
|
|
this.waiting.push(call);
|
|
this.launchCovered();
|
|
} else this.held.push(call);
|
|
return;
|
|
}
|
|
const entry = this.replayed[index]!;
|
|
if (entry.global !== undefined) {
|
|
if (call.target !== "global" || call.name !== entry.global.name || !sameJson(call.args, entry.global.args)) {
|
|
this.diverge(`call ${call.id} is ${describe(call)}, but was a call to ${entry.global.name}`);
|
|
return;
|
|
}
|
|
this.ready.set(call.id, entry.global.outcome as CodemodeOutcome);
|
|
void this.gate.then(() => this.deliverReplayed());
|
|
return;
|
|
}
|
|
if (call.target !== "tool") {
|
|
this.diverge(`call ${call.id} is ${describe(call)}, but was a tool call`);
|
|
return;
|
|
}
|
|
void this.gate
|
|
.then(() => this.callTool(call))
|
|
.then(
|
|
(outcome) => {
|
|
this.ready.set(call.id, outcome);
|
|
this.deliverReplayed();
|
|
},
|
|
(error: unknown) => {
|
|
if (this.context.abortSignal?.aborted) this.fail(error);
|
|
// The key names another tool or other arguments.
|
|
else this.diverge(`call ${call.id} is ${describe(call)}: ${errorMessage(error)}`);
|
|
},
|
|
);
|
|
}
|
|
|
|
/** Replay: deliver journaled results in journal order, each once the VM has issued its call. */
|
|
private deliverReplayed(): void {
|
|
while (this.nextReplayed < this.replayed.length && !this.stopped) {
|
|
const entry = this.replayed[this.nextReplayed]!;
|
|
const outcome = this.ready.get(entry.id);
|
|
if (entry.id > this.issued || outcome === undefined) return;
|
|
this.ready.delete(entry.id);
|
|
this.nextReplayed++;
|
|
this.run.deliver(this.nextReplayed, entry.at, entry.id, outcome);
|
|
}
|
|
}
|
|
|
|
private onIdle(seq: number, issued: number): void {
|
|
if (this.stopped) return;
|
|
// A count an earlier attempt recorded must match; a missing one is recorded now.
|
|
if (this.rerun && seq <= this.replayed.length) {
|
|
const recorded = seq === 0 ? this.firstIssued : this.replayed[seq - 1]!.issuedAfter;
|
|
if (recorded !== undefined && recorded !== issued) {
|
|
this.diverge(
|
|
seq === 0
|
|
? `its first run made ${issued} calls, where it had made ${recorded}`
|
|
: `after result ${seq} it had made ${issued} calls, where it had made ${recorded}`,
|
|
);
|
|
return;
|
|
}
|
|
}
|
|
this.acks.set(seq, issued);
|
|
if (!this.replaying) {
|
|
// A rerun without a journal: every call an earlier attempt could have admitted is made by now.
|
|
if (seq === 0 && this.rerun) void this.release();
|
|
// Committed at once when the script has processed everything delivered: calls it made wait for it, and a script
|
|
// waiting on a long call is where a restart usually finds it. Otherwise with the next commit; the VM catches up
|
|
// without waiting on anything.
|
|
if (seq === this.journalLength) void this.pump();
|
|
return;
|
|
}
|
|
// Caught up: the VM processed everything delivered and waits.
|
|
if (seq !== this.nextReplayed) return;
|
|
const next = this.replayed[seq];
|
|
if (next !== undefined) {
|
|
if (next.id > issued) this.diverge(`it waits without making call ${next.id}, which it made before`);
|
|
return;
|
|
}
|
|
// The last replayed result is acknowledged: continue live, admitting the calls the journal did not have once their
|
|
// count is committed.
|
|
this.replaying = false;
|
|
void this.pump();
|
|
void this.release();
|
|
}
|
|
|
|
/**
|
|
* Let the held calls launch, and later live calls, each once its issued count is committed. On a rerun, an earlier
|
|
* attempt may have admitted some of them under their keys, as other calls: when the replay releases (idle after the
|
|
* last journaled result, or after its first run without a journal) its counts matched every recorded one, so it has
|
|
* made every call an earlier attempt could have admitted. All held calls are checked first, whatever their target,
|
|
* and a divergence launches none. Calls made after this are new.
|
|
*/
|
|
private async release(): Promise<void> {
|
|
await this.gate;
|
|
if (this.stopped || this.released) return;
|
|
const calls = this.held.splice(0);
|
|
if (this.rerun) {
|
|
for (const call of calls) {
|
|
let other: string | undefined;
|
|
try {
|
|
other = await this.earlier(call);
|
|
} catch (error) {
|
|
this.fail(error);
|
|
return;
|
|
}
|
|
if (this.stopped) return;
|
|
if (other !== undefined) {
|
|
this.diverge(`call ${call.id} is ${describe(call)}, but was ${other}`);
|
|
return;
|
|
}
|
|
}
|
|
}
|
|
this.released = true;
|
|
this.waiting.push(...calls, ...this.held.splice(0));
|
|
this.launchCovered();
|
|
}
|
|
|
|
/** Launch the waiting calls whose issued count is committed. */
|
|
private launchCovered(): void {
|
|
if (!this.released || this.stopped) return;
|
|
const covered = this.waiting.filter((call) => call.id <= this.durableIssued);
|
|
if (covered.length === 0) return;
|
|
this.waiting = this.waiting.filter((call) => call.id > this.durableIssued);
|
|
for (const call of covered) this.launch(call);
|
|
}
|
|
|
|
/** The call an earlier attempt made under this call's key, when it is another one: another target, tool, or arguments. */
|
|
private async earlier(call: CodemodeHostCall): Promise<string | undefined> {
|
|
const { api, context } = this;
|
|
const taskId = (await api.snapshot(NestedCallDoc, api.taskId, `c${call.id}`, context))?.taskId;
|
|
if (taskId === undefined) return undefined;
|
|
const input = (await api.getTask(taskId, context))?.input as ToolTaskInput | undefined;
|
|
if (input?.kind !== "nested") return undefined;
|
|
const made = input.call;
|
|
if (call.target === "tool" && made.name === call.name && sameJson(made.arguments, call.args ?? {}))
|
|
return undefined;
|
|
return `tools.${made.name}(${JSON.stringify(made.arguments)})`;
|
|
}
|
|
|
|
/** Run one live call and journal its result. */
|
|
private launch(call: CodemodeHostCall): void {
|
|
if (this.stopped) return;
|
|
const global = call.target === "global" ? this.options.globals?.find((g) => g.name === call.name) : undefined;
|
|
const running: Promise<{ outcome: CodemodeOutcome; global?: JournalEntry["global"] }> =
|
|
call.target === "tool"
|
|
? this.callTool(call).then((outcome) => ({ outcome }))
|
|
: this.callGlobal(call, global).then((outcome) => ({
|
|
outcome,
|
|
global: {
|
|
name: call.name,
|
|
...(call.args === undefined ? {} : { args: call.args as JsonValue }),
|
|
outcome: outcome as JsonObject,
|
|
},
|
|
}));
|
|
running.then(
|
|
(done) => {
|
|
this.queue.push({ id: call.id, at: Date.now(), ...done });
|
|
void this.pump();
|
|
},
|
|
(error: unknown) => {
|
|
if (this.context.abortSignal?.aborted) this.fail(error);
|
|
// A key an earlier attempt admitted with another tool or other arguments: this attempt took another path.
|
|
else this.diverge(`call ${call.id} is ${describe(call)}: ${errorMessage(error)}`);
|
|
},
|
|
);
|
|
}
|
|
|
|
private async callTool(call: CodemodeHostCall): Promise<CodemodeOutcome> {
|
|
// Passed as the script gave them; validation reports arguments that are not an object.
|
|
const args = (call.args ?? {}) as JsonObject;
|
|
const result = await this.api.executeTool(call.name, args, this.context, { key: `c${call.id}` });
|
|
return outcomeOf(call.name, result);
|
|
}
|
|
|
|
private async callGlobal(call: CodemodeHostCall, global: CodemodeGlobal | undefined): Promise<CodemodeOutcome> {
|
|
if (global === undefined) return { ok: false, message: `Unknown global ${call.name}` };
|
|
try {
|
|
// `undefined` stays undefined: the journal omits the value, and a replay resolves with undefined again.
|
|
return { ok: true, value: await global.execute(call.args, this.api, this.context) };
|
|
} catch (error) {
|
|
if (this.context.abortSignal?.aborted) throw error;
|
|
return { ok: false, message: errorMessage(error) };
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Journal finished calls and acknowledged counts in one commit, then deliver the calls and launch the calls the
|
|
* counts cover; one commit in flight at a time, so what arrives meanwhile goes into the next.
|
|
*/
|
|
private async pump(): Promise<void> {
|
|
if (this.committing || this.stopped) return;
|
|
if (this.queue.length === 0 && this.acks.size === 0) return;
|
|
this.committing = true;
|
|
const batch = this.queue.splice(0);
|
|
const acks = [...this.acks];
|
|
this.acks.clear();
|
|
try {
|
|
await this.api.commit(async (tx) => {
|
|
const journal = await tx.doc(CodemodeJournalDoc, this.api.taskId);
|
|
for (const [seq, issued] of acks) {
|
|
if (seq === 0) journal.firstIssued ??= issued;
|
|
else {
|
|
const entry = journal.entries[seq - 1];
|
|
if (entry !== undefined && entry.issuedAfter === undefined) entry.issuedAfter = issued;
|
|
}
|
|
}
|
|
for (const item of batch) {
|
|
const entry = {
|
|
id: item.id,
|
|
at: item.at,
|
|
...(item.global === undefined ? {} : { global: item.global }),
|
|
};
|
|
journal.entries.push(
|
|
copyJson(entry as unknown as JsonValue, { omitUndefinedProperties: true }) as unknown as JournalEntry,
|
|
);
|
|
}
|
|
}, this.context);
|
|
} catch (error) {
|
|
this.fail(error);
|
|
return;
|
|
}
|
|
for (const [, issued] of acks) this.durableIssued = Math.max(this.durableIssued, issued);
|
|
for (const item of batch) {
|
|
this.journalLength++;
|
|
if (!this.run.finished) this.run.deliver(this.journalLength, item.at, item.id, item.outcome);
|
|
}
|
|
this.launchCovered();
|
|
this.committing = false;
|
|
void this.pump();
|
|
}
|
|
}
|
|
|
|
function describe(call: CodemodeHostCall): string {
|
|
return `${call.target === "tool" ? "tools." : ""}${call.name}(${JSON.stringify(call.args) ?? ""})`;
|
|
}
|
|
|
|
/** Structural JSON equality; key order does not matter. */
|
|
function sameJson(a: unknown, b: unknown): boolean {
|
|
if (a === b) return true;
|
|
if (typeof a !== "object" || typeof b !== "object" || a === null || b === null) return false;
|
|
if (Array.isArray(a) || Array.isArray(b)) {
|
|
return (
|
|
Array.isArray(a) && Array.isArray(b) && a.length === b.length && a.every((item, i) => sameJson(item, b[i]))
|
|
);
|
|
}
|
|
const keys = Object.keys(a);
|
|
return (
|
|
keys.length === Object.keys(b).length &&
|
|
keys.every(
|
|
(key) =>
|
|
Object.hasOwn(b, key) && sameJson((a as Record<string, unknown>)[key], (b as Record<string, unknown>)[key]),
|
|
)
|
|
);
|
|
}
|