From 0fb1aee749fbff729f8f373f3841d787779bebe0 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Sun, 11 Oct 2026 19:50:17 +0000 Subject: [PATCH] feat(codemode): deterministic runs that the host answers (sandbox.start) For durable code mode: a host can rerun a script after a restart and get the same calls with the same arguments. start() runs the script with a virtual clock (startedAt, then each result's time, never backwards), randomness from a seed, UTC, no WeakRef/FinalizationRegistry, and the run's own tools and globals; every call goes to onCall with the VM's call id, and the host answers with run.deliver(seq, now, id, outcome). The VM applies results strictly in seq order whatever the transport does, reports its fingerprint (engine versions, prelude hash, limits) before the script runs, the calls issued after each result (onIdle), and the result it was processing when the script ended (result.during). Rejections can carry properties for the script's Error. execute() is unchanged. --- packages/codemode/CHANGELOG.md | 1 + packages/codemode/README.md | 31 ++ packages/codemode/src/index.ts | 7 + packages/codemode/src/portable.ts | 7 + packages/codemode/src/runtime/execution.ts | 196 +++++++-- packages/codemode/src/runtime/inline.ts | 1 + .../codemode/src/runtime/prelude-source.ts | 14 +- packages/codemode/src/runtime/protocol.ts | 51 ++- packages/codemode/src/runtime/remote.ts | 5 + packages/codemode/src/runtime/vm.ts | 161 ++++++- packages/codemode/src/types.ts | 90 +++- packages/codemode/test/deterministic.test.ts | 400 ++++++++++++++++++ 12 files changed, 904 insertions(+), 60 deletions(-) create mode 100644 packages/codemode/test/deterministic.test.ts diff --git a/packages/codemode/CHANGELOG.md b/packages/codemode/CHANGELOG.md index 8e7e6798d..82215ae75 100644 --- a/packages/codemode/CHANGELOG.md +++ b/packages/codemode/CHANGELOG.md @@ -7,6 +7,7 @@ - Added `RemoteCodemodeSandbox`, which runs each script's VM in another isolate while tools run in the caller. `CodemodeSandboxDurableObject` from the new `@earendil-works/pi-codemode/cloudflare` entry is its remote end, deployed as its own Cloudflare Worker. `examples/cloudflare` shows the setup - Added `InlineCodemodeSandbox`, which runs the VM on the calling thread for runtimes without worker threads. Its `interruptBudget` limits scripts that compute without awaiting a tool - Added the `@earendil-works/pi-codemode/portable` entry, which has no Node imports, and `CodemodeSandboxBase`, the API all three sandboxes share +- Added `sandbox.start()`, a deterministic run whose calls the host answers itself (`onCall`, `run.deliver()`), so a host can replay a script after a restart: a virtual clock from `startedAt`, randomness from `seed`, UTC, no `WeakRef`/`FinalizationRegistry`, results applied in the order the host numbers them, the VM's fingerprint (`onStarted`), and the calls issued after each result (`onIdle`). Works with every sandbox; `execute()` is unchanged ### Changed diff --git a/packages/codemode/README.md b/packages/codemode/README.md index c78798ba3..fedcbdf85 100644 --- a/packages/codemode/README.md +++ b/packages/codemode/README.md @@ -293,6 +293,37 @@ const codemodeTool: AgentTool = { `result.output` items already have the shape of `@earendil-works/pi-ai`'s `TextContent` and `ImageContent`. Calling `tool.execute()` directly skips the agent's `beforeToolCall` and `afterToolCall` hooks. To apply them to nested calls too, run each call through `runToolCall()` from `@earendil-works/pi-agent-core`, as the [mcp-codemode example](https://github.com/earendil-works/pi/tree/main/packages/agent/examples/mcp-codemode) does. That example also rejects failed nested calls inside the script and combines codemode with MCP tools. +## Deterministic runs + +`sandbox.start(code, options)` runs a script so that a host can run it again after a restart and get the same behavior, for example to resume it from a durable log of its calls' results. `execute()` is unaffected. + +The VM only runs when the host hands it something, and has no timers or I/O, so it is deterministic except for what it reads from its environment. `start()` fixes that: + +- **Time:** `Date.now()` reads `startedAt`, then the `now` of the result being processed, never going backwards. `performance.now()` counts from `startedAt`. +- **Randomness:** `Math.random()` and the VM's other randomness come from `seed`. +- **Timezone:** UTC, whatever the host's. +- **Garbage collection:** `WeakRef` and `FinalizationRegistry` do not exist. +- **Result order:** the host answers each call with a result number (1, 2, ...), and the VM applies results strictly in that order, whatever order a transport delivers them in. + +The run declares its own tools and globals, and the host answers every call itself: + +```ts +const run = sandbox.start(code, { + startedAt: Date.now(), + seed: 7, + tools: [{ name: "read", description: "Read a file" }], + onStarted: (fingerprint) => {}, // what the VM is: engine versions, prelude hash, limits + onCall: (call) => { + // call.id is the VM's own number for the call, in the order the script made them. + void readFile(call.args).then((text) => run.deliver(nextResult++, Date.now(), call.id, { ok: true, value: text })); + }, + onIdle: (seq, issued) => {}, // after result `seq` (0: the first run), `issued` calls in total +}); +const result = await run.result; +``` + +A run has no deadline unless `timeoutMs` is given. Result numbers must run 1, 2, ... without gaps. The same script, `startedAt`, `seed`, tools (with their `jsName`), globals and store, answered with the same results in the same order, makes the same calls with the same arguments and produces the same output on any VM with the same fingerprint. To end a run with an error the script cannot catch, abort its `signal`. + ## Results `execute()` never rejects for script failures. `result.error.kind` is one of: diff --git a/packages/codemode/src/index.ts b/packages/codemode/src/index.ts index c5d2536ce..64a13b9ee 100644 --- a/packages/codemode/src/index.ts +++ b/packages/codemode/src/index.ts @@ -45,12 +45,19 @@ export { export type { CodemodeCall, CodemodeCallStatus, + CodemodeDeclaredGlobal, + CodemodeDeclaredTool, + CodemodeDuring, CodemodeError, CodemodeErrorKind, CodemodeExecuteOptions, + CodemodeHostCall, CodemodeJsonSchema, + CodemodeOutcome, CodemodeOutputItem, CodemodeResult, + CodemodeRun, + CodemodeRunOptions, CodemodeSandboxBaseOptions, CodemodeSandboxOptions, CodemodeStoreWrites, diff --git a/packages/codemode/src/portable.ts b/packages/codemode/src/portable.ts index f6f366f0a..27069de33 100644 --- a/packages/codemode/src/portable.ts +++ b/packages/codemode/src/portable.ts @@ -32,12 +32,19 @@ export { export type { CodemodeCall, CodemodeCallStatus, + CodemodeDeclaredGlobal, + CodemodeDeclaredTool, + CodemodeDuring, CodemodeError, CodemodeErrorKind, CodemodeExecuteOptions, + CodemodeHostCall, CodemodeJsonSchema, + CodemodeOutcome, CodemodeOutputItem, CodemodeResult, + CodemodeRun, + CodemodeRunOptions, CodemodeSandboxBaseOptions, CodemodeStoreWrites, CodemodeTool, diff --git a/packages/codemode/src/runtime/execution.ts b/packages/codemode/src/runtime/execution.ts index dfb681cbb..d1a137006 100644 --- a/packages/codemode/src/runtime/execution.ts +++ b/packages/codemode/src/runtime/execution.ts @@ -25,10 +25,16 @@ import { toCodemodeIdentifier } from "../identifier.ts"; import type { CodemodeCall, CodemodeCallStatus, + CodemodeDeclaredGlobal, + CodemodeDeclaredTool, + CodemodeDuring, CodemodeError, CodemodeExecuteOptions, + CodemodeOutcome, CodemodeOutputItem, CodemodeResult, + CodemodeRun, + CodemodeRunOptions, CodemodeSandboxBaseOptions, CodemodeStoreWrites, CodemodeTool, @@ -142,14 +148,39 @@ interface PendingCall { controller: AbortController; } +/** Checks global names: identifiers or `namespace.member`, not shadowing a built-in global or each other. */ +function checkGlobals(globals: readonly CodemodeDeclaredGlobal[]): void { + const names = new Set(); + const namespaces = new Set(); + for (const global of globals) { + const parts = global.name.split("."); + if (parts.length > 2 || !parts.every((part) => IDENTIFIER.test(part)) || RESERVED_GLOBALS.has(parts[0])) { + throw new Error(`Invalid global name "${global.name}"`); + } + if (names.has(global.name)) throw new Error(`Global "${global.name}" is already registered`); + if (parts.length === 2) namespaces.add(parts[0]); + names.add(global.name); + } + for (const name of namespaces) { + if (names.has(name)) throw new Error(`Global "${name}" conflicts with the namespace "${name}"`); + } +} + +/** Callbacks of a host-controlled run: its calls go to the caller, which answers them with `deliver()`. */ +type Control = Pick; + interface ExecutionOptions { code: string; - tools: ReadonlyMap; - globals: ReadonlyMap; + tools: readonly CodemodeDeclaredTool[]; + globals: readonly CodemodeDeclaredGlobal[]; + /** Run the script's calls; unused when `control` is set. */ + executors: { tools: ReadonlyMap; globals: ReadonlyMap }; timeoutMs: number; signal: AbortSignal | undefined; memoryLimitBytes: number | undefined; store: Record; + deterministic?: { startedAt: number; seed: number }; + control?: Control; launch: VmLauncher; } @@ -160,8 +191,10 @@ class Execution { private channel: VmChannel | undefined; /** Replies produced before the launcher returned the channel. */ private unsent: HostToWorkerMessage[] = []; - private readonly tools: ReadonlyMap; - private readonly globals: ReadonlyMap; + private readonly executors: ExecutionOptions["executors"]; + private readonly control: Control | undefined; + /** Result numbers `deliver()` used. */ + private readonly delivered = new Set(); private readonly signal: AbortSignal | undefined; private readonly timer: ReturnType | undefined; private readonly output: CodemodeOutputItem[] = []; @@ -173,8 +206,8 @@ class Execution { this.promise = new Promise((resolve) => { this.resolveResult = resolve; }); - this.tools = options.tools; - this.globals = options.globals; + this.executors = options.executors; + this.control = options.control; this.signal = options.signal; if (Number.isFinite(options.timeoutMs)) { @@ -194,17 +227,15 @@ class Execution { const data: VmStartData = { code: options.code, - tools: [...options.tools.values()].map((tool) => ({ + tools: options.tools.map((tool) => ({ name: tool.name, - jsName: toCodemodeIdentifier(tool.name), + jsName: tool.jsName ?? toCodemodeIdentifier(tool.name), description: tool.description ?? "", })), - globals: [...options.globals.values()].map((global) => ({ - name: global.name, - spread: global.spread === true, - })), + globals: options.globals.map((global) => ({ name: global.name, spread: global.spread === true })), memoryLimitBytes: options.memoryLimitBytes, store: options.store, + ...(options.deterministic ? { deterministic: options.deterministic } : {}), }; const events: VmEvents = { message: (message) => this.receive(message), @@ -225,6 +256,46 @@ class Execution { ); } + /** `CodemodeRun.deliver()`. */ + deliver(seq: number, now: number, id: number, outcome: CodemodeOutcome): void { + if (this.finished) return; + if (!Number.isFinite(now)) throw new Error(`The clock value of result ${seq} is not a finite number`); + if (!Number.isInteger(seq) || seq < 1 || this.delivered.has(seq)) { + throw new Error(`Result number ${seq} is not a positive integer used once`); + } + const pending = this.pending.get(id); + if (pending === undefined) throw new Error(`Call ${id} is not waiting for a result`); + if (!outcome.ok && outcome.properties !== undefined) { + const { properties } = outcome; + if (typeof properties !== "object" || properties === null || Array.isArray(properties)) { + throw new Error(`The properties of result ${seq} are not an object`); + } + } + // Serialized before anything is recorded, so a value JSON cannot hold leaves the call waiting for another answer. + const payload = outcome.ok + ? outcome.value === undefined + ? undefined + : JSON.stringify(outcome.value) + : outcome.message; + const properties = + !outcome.ok && outcome.properties !== undefined ? JSON.stringify(outcome.properties) : undefined; + this.delivered.add(seq); + this.pending.delete(id); + if (pending.record) { + pending.record.status = outcome.ok ? "ok" : "error"; + pending.record.durationMs = performance.now() - pending.startedAt; + } + this.post({ + type: "result", + id, + ok: outcome.ok, + payload, + seq, + now, + ...(properties === undefined ? {} : { properties }), + }); + } + abort(message: string): Promise { this.finish({ kind: "aborted", message }); return this.promise; @@ -264,7 +335,14 @@ class Execution { break; case "call": if (this.pending.has(message.id)) throw new BridgeError(`duplicate call id ${message.id}`); - void this.handleCall(message); + if (this.control) this.forwardCall(message, this.control); + else void this.handleCall(message); + break; + case "started": + this.control?.onStarted?.(message.fingerprint); + break; + case "idle": + this.control?.onIdle?.(message.seq, message.issued); break; case "done": this.handleDone(message); @@ -278,11 +356,22 @@ class Execution { private handleDone(message: Extract): void { // Decode everything before finish(), which must not throw once it starts. if (!message.ok) { - this.finish(parseScriptError(message.error)); + this.finish(parseScriptError(message.error), undefined, undefined, message.during); return; } const value = message.value === undefined ? undefined : parseBridgeJson(message.value, "return value"); - this.finish(undefined, value, parseStoreWrites(message.writes)); + this.finish(undefined, value, parseStoreWrites(message.writes), message.during); + } + + /** A host-controlled run: record the call and hand it to the caller, which answers with `deliver()`. */ + private forwardCall(message: Extract, control: Control): void { + const { id, name, target } = message; + const args = message.args === undefined ? undefined : parseBridgeJson(message.args, "call arguments"); + const record: CodemodeCall | undefined = + target === "tool" ? { name, status: "cancelled", durationMs: 0 } : undefined; + if (record) this.calls.push(record); + this.pending.set(id, { record, startedAt: performance.now(), controller: new AbortController() }); + control.onCall({ id, target, name, args }); } private async handleCall(message: Extract): Promise { @@ -296,7 +385,7 @@ class Execution { let status: CodemodeCallStatus; let reply: HostToWorkerMessage; try { - const tool = (isTool ? this.tools : this.globals).get(name); + const tool = (isTool ? this.executors.tools : this.executors.globals).get(name); if (!tool) throw new Error(`Unknown ${isTool ? "tool" : "global"} "${name}"`); const args: unknown = message.args === undefined ? undefined : JSON.parse(message.args); const value = await tool.execute(args, { signal: pending.controller.signal }); @@ -317,7 +406,12 @@ class Execution { this.post(reply); } - private finish(error: CodemodeError | undefined, value?: unknown, writes?: CodemodeStoreWrites): void { + private finish( + error: CodemodeError | undefined, + value?: unknown, + writes?: CodemodeStoreWrites, + during?: CodemodeDuring, + ): void { if (this.finished) return; this.finished = true; clearTimeout(this.timer); @@ -331,14 +425,16 @@ class Execution { this.pending.clear(); this.unsent = []; + const ended = during === undefined ? {} : { during: { seq: during.seq, issued: during.issued } }; const result: CodemodeResult = error - ? { ok: false, error, output: this.output, calls: this.calls } + ? { ok: false, error, output: this.output, calls: this.calls, ...ended } : { ok: true, value, output: this.output, calls: this.calls, storeWrites: writes ?? { set: {}, delete: [] }, + ...ended, }; if (!this.channel) { this.resolveResult(result); @@ -374,19 +470,8 @@ export class CodemodeSandboxBase { this.memoryLimitBytes = options.memoryLimitBytes; this.launch = launch; for (const tool of options.tools ?? []) this.registerTool(tool); - const namespaces = new Set(); - for (const global of options.globals ?? []) { - const parts = global.name.split("."); - if (parts.length > 2 || !parts.every((part) => IDENTIFIER.test(part)) || RESERVED_GLOBALS.has(parts[0])) { - throw new Error(`Invalid global name "${global.name}"`); - } - if (this.globalsByName.has(global.name)) throw new Error(`Global "${global.name}" is already registered`); - if (parts.length === 2) namespaces.add(parts[0]); - this.globalsByName.set(global.name, global); - } - for (const name of namespaces) { - if (this.globalsByName.has(name)) throw new Error(`Global "${name}" conflicts with the namespace "${name}"`); - } + checkGlobals(options.globals ?? []); + for (const global of options.globals ?? []) this.globalsByName.set(global.name, global); } /** Throws if a tool with the same name is already registered. */ @@ -414,10 +499,12 @@ export class CodemodeSandboxBase { */ execute(code: string, options: CodemodeExecuteOptions = {}): Promise { if (this.closed) return Promise.reject(new Error("Sandbox is closed")); + const tools = new Map(this.toolsByName); const execution = new Execution({ code, - tools: new Map(this.toolsByName), - globals: this.globalsByName, + tools: [...tools.values()], + globals: [...this.globalsByName.values()], + executors: { tools, globals: this.globalsByName }, timeoutMs: options.timeoutMs ?? this.timeoutMs, signal: options.signal, memoryLimitBytes: this.memoryLimitBytes, @@ -428,6 +515,49 @@ export class CodemodeSandboxBase { return execution.promise.finally(() => this.running.delete(execution)); } + /** + * Run `code` deterministically, with the caller answering its calls: the same script, start time, seed, tools, + * globals, and store, answered with the same results in the same `seq` order, makes the same calls with the same + * arguments and produces the same output, on any VM with the same fingerprint. A host can therefore replay it + * after a restart. The VM runs in UTC, its clock reads `startedAt` and then the `now` of the result being + * processed, its randomness comes from `seed`, and `WeakRef` and `FinalizationRegistry` do not exist. + * + * The sandbox's own tools and globals are not used: the run declares its own, and every call goes to + * `options.onCall`. Never rejects for script failures; abort through `options.signal` to end the run with an + * error the script cannot catch. + */ + start(code: string, options: CodemodeRunOptions): CodemodeRun { + if (this.closed) throw new Error("Sandbox is closed"); + if (!Number.isFinite(options.startedAt) || options.startedAt < 0) { + throw new Error("startedAt must be a non-negative number of milliseconds since the epoch"); + } + if (!Number.isInteger(options.seed) || options.seed < 0 || options.seed > 0xffffffff) { + throw new Error("seed must be a 32-bit unsigned integer"); + } + const tools = options.tools ?? []; + if (new Set(tools.map((tool) => tool.name)).size !== tools.length) throw new Error("Tool names must be unique"); + const scriptNames = tools.map((tool) => tool.jsName ?? toCodemodeIdentifier(tool.name)); + if (new Set(scriptNames).size !== tools.length) throw new Error("Tool script names must be unique"); + const globals = options.globals ?? []; + checkGlobals(globals); + const execution = new Execution({ + code, + tools, + globals, + executors: { tools: new Map(), globals: new Map() }, + timeoutMs: options.timeoutMs ?? Number.POSITIVE_INFINITY, + signal: options.signal, + memoryLimitBytes: this.memoryLimitBytes, + store: serializeStore(options.store), + deterministic: { startedAt: options.startedAt, seed: options.seed }, + control: { onStarted: options.onStarted, onCall: options.onCall, onIdle: options.onIdle }, + launch: this.launch, + }); + this.running.add(execution); + const result = execution.promise.finally(() => this.running.delete(execution)); + return { result, deliver: (seq, now, id, outcome) => execution.deliver(seq, now, id, outcome) }; + } + /** Aborts in-flight executions (they resolve with `kind: "aborted"`) and rejects new ones. */ async close(): Promise { this.closed = true; diff --git a/packages/codemode/src/runtime/inline.ts b/packages/codemode/src/runtime/inline.ts index ddee48eae..dd183a6c5 100644 --- a/packages/codemode/src/runtime/inline.ts +++ b/packages/codemode/src/runtime/inline.ts @@ -73,6 +73,7 @@ export function inlineLauncher( data, wasm: module, maxStackSize: INLINE_MAX_STACK_SIZE, + interruptBudget, shouldInterrupt: () => { if (stopped || settled || exhausted) return true; if (++polls <= interruptBudget) return false; diff --git a/packages/codemode/src/runtime/prelude-source.ts b/packages/codemode/src/runtime/prelude-source.ts index 491608d88..d7477423f 100644 --- a/packages/codemode/src/runtime/prelude-source.ts +++ b/packages/codemode/src/runtime/prelude-source.ts @@ -17,6 +17,8 @@ * with a successful "done". * * Evaluates to a function `(bridge, toolsJson, globalsJson, storeJson) => { settle, run, stalled }`. + * `settle(id, ok, payload, propertiesJson)` settles call `id`; a rejection's `Error` gets the own properties of the + * `propertiesJson` object, except `name`, `message`, and `stack`. * `toolsJson` lists `{ name, jsName, description }`: `tools[jsName]` and `tools[name]` call the * tool, and `ALL_TOOLS` lists `{ name: jsName, description }`. * `stalled()` reports a script that has not finished while no host call is pending: with no timers @@ -454,12 +456,20 @@ export const PRELUDE_SOURCE: string = `(function (bridge, toolsJson, globalsJson Object.defineProperty(globalThis, "exit", { value: exit, enumerable: true }); return { - settle(id, ok, payload) { + settle(id, ok, payload, propertiesJson) { const entry = pending.get(id); if (!entry) return; pending.delete(id); if (!ok) { - entry.reject(new ErrorCtor(payload)); + const error = new ErrorCtor(payload); + if (propertiesJson !== undefined) { + const properties = parse(propertiesJson); + for (const key of Object.keys(properties)) { + if (key === "name" || key === "message" || key === "stack") continue; + Object.defineProperty(error, key, { value: properties[key], writable: true, enumerable: true, configurable: true }); + } + } + entry.reject(error); return; } let value; diff --git a/packages/codemode/src/runtime/protocol.ts b/packages/codemode/src/runtime/protocol.ts index e93248760..0f1e126a4 100644 --- a/packages/codemode/src/runtime/protocol.ts +++ b/packages/codemode/src/runtime/protocol.ts @@ -1,4 +1,4 @@ -import type { CodemodeOutputItem } from "../types.ts"; +import type { CodemodeDuring, CodemodeOutputItem } from "../types.ts"; import type { CodemodeWasmModule } from "../wasm.ts"; /** @@ -19,6 +19,12 @@ export interface VmStartData { memoryLimitBytes: number | undefined; /** Snapshot for `load()`: key to JSON text. */ store: Record; + /** + * Run deterministically (`CodemodeSandboxBase.start()`): UTC, a virtual clock starting at `startedAt` (ms since + * the epoch) that moves only with delivered results, randomness from `seed`, no `WeakRef`, results applied in + * `seq` order, and `started`/`idle` messages. + */ + deterministic?: { startedAt: number; seed: number }; } export interface WorkerData extends VmStartData { @@ -37,20 +43,49 @@ export type ScriptErrorJson = string; export type WorkerToHostMessage = | { type: "call"; id: number; target: "tool" | "global"; name: string; args: string | undefined } | { type: "output"; item: CodemodeOutputItem } - /** `writes` is a JSON array of `[key, json]` for `store()` and `[key]` for deletions. */ - | { type: "done"; ok: true; value: string | undefined; writes: string } - | { type: "done"; ok: false; error: ScriptErrorJson } + /** + * `writes` is a JSON array of `[key, json]` for `store()` and `[key]` for deletions. Deterministic mode: `during` + * is the result the VM was processing when the script finished (0: its first run) and the calls issued by then. + */ + | { type: "done"; ok: true; value: string | undefined; writes: string; during?: CodemodeDuring } + | { type: "done"; ok: false; error: ScriptErrorJson; during?: CodemodeDuring } /** The VM failed outside the script's control, for example a wasm trap. */ - | { type: "crash"; message: string }; + | { type: "crash"; message: string } + /** Deterministic mode, before the script runs: what the VM is, so a replay can refuse a different one. */ + | { type: "started"; fingerprint: string } + /** + * Deterministic mode, after the script waits again: having processed result `seq` (0: the script's first run), it + * has issued `issued` calls in total. + */ + | { type: "idle"; seq: number; issued: number }; export type HostToWorkerMessage = - /** `payload` is the JSON result when `ok`, otherwise the error message. */ - { type: "result"; id: number; ok: boolean; payload: string | undefined }; + /** + * `payload` is the JSON result when `ok`, otherwise the error message. Deterministic mode: `seq` numbers the + * results from 1 in the order the VM applies them, and the VM's clock reads `now` while it processes this one. + */ + { + type: "result"; + id: number; + ok: boolean; + payload: string | undefined; + seq?: number; + now?: number; + /** Deterministic mode, `ok: false`: a JSON object whose properties the script's `Error` gets. */ + properties?: string; + }; export function isWorkerToHostMessage(value: unknown): value is WorkerToHostMessage { if (typeof value !== "object" || value === null) return false; const type = (value as { type?: unknown }).type; - return type === "call" || type === "output" || type === "done" || type === "crash"; + return ( + type === "call" || + type === "output" || + type === "done" || + type === "crash" || + type === "started" || + type === "idle" + ); } export function isHostToWorkerMessage(value: unknown): value is HostToWorkerMessage { diff --git a/packages/codemode/src/runtime/remote.ts b/packages/codemode/src/runtime/remote.ts index 022f76328..f5f02de26 100644 --- a/packages/codemode/src/runtime/remote.ts +++ b/packages/codemode/src/runtime/remote.ts @@ -163,7 +163,12 @@ export function serveCodemodeRemote( } break; } + case "started": + buffered.push(typed); + break; + // Sent at once: a host replaying a script waits for it to see that the script issued no further call. case "call": + case "idle": exchange([...takeBuffer(), typed]).then(reply, () => end(undefined)); break; default: diff --git a/packages/codemode/src/runtime/vm.ts b/packages/codemode/src/runtime/vm.ts index d93423365..2618eef9b 100644 --- a/packages/codemode/src/runtime/vm.ts +++ b/packages/codemode/src/runtime/vm.ts @@ -19,8 +19,23 @@ * * Failures after the VM exists are reported as a `crash` message; `startVm` only rejects when the * VM cannot be created at all. + * + * Deterministic mode (`data.deterministic`, used by `CodemodeSandboxBase.start()`): a script must make the same + * calls with the same arguments every time it runs with the same results, so a host can replay it after a restart. + * The VM is otherwise deterministic, because it only runs when the host delivers something and has no timers. + * What remains is fixed here: + * - Time: every WASI clock reads virtual time, `startedAt` and then the `now` of the result being processed, never going backwards. + * While the context is created, the realtime clock reads a value derived from the seed instead, because QuickJS + * seeds `Math.random()` from the first clock read, and which read comes first is the engine's business. + * - Randomness: `random_get` draws from a PRNG over the seed. + * - Timezone: UTC, not the host's. + * - Garbage collection: `WeakRef` and `FinalizationRegistry` are not available; collection timing is not. + * - Delivery order: results carry a `seq` and are applied strictly in that order, buffering early ones, because a + * transport (RPC replies) may deliver them in another order than the host sent them. + * The VM reports `started` with its fingerprint before the script runs, and `idle` with the number of calls issued + * after each drain, so the host can check a replay step by step. */ -import { JSException, type JSValueHandle, MAX_STACK_SIZE, QuickJS } from "quickjs-wasi"; +import { Intrinsics, JSException, type JSValueHandle, MAX_STACK_SIZE, QuickJS } from "quickjs-wasi"; import type { CodemodeWasmModule } from "../wasm.ts"; import { PRELUDE_SOURCE } from "./prelude-source.ts"; import type { HostToWorkerMessage, VmStartData, WorkerToHostMessage } from "./protocol.ts"; @@ -41,10 +56,74 @@ export interface StartVmOptions { * (512 KiB), which fits the 4 MB stack of a Node worker thread but not a 1 MB main thread. */ maxStackSize?: number; + /** Interrupt polls this VM may use, if the launcher counts them; part of the fingerprint. */ + interruptBudget?: number; post(message: WorkerToHostMessage): void; shouldInterrupt(): boolean; } +/** + * Bumped whenever this module or the prelude changes what a deterministic script observes, beyond what the + * fingerprint's other parts (engine versions, prelude hash, limits) already capture. + */ +const DETERMINISTIC_VERSION = 1; + +/** FNV-1a over UTF-16 code units, as 8 hex digits: identifies the prelude in the fingerprint. */ +function fnv1a(text: string): string { + let hash = 0x811c9dc5; + for (let index = 0; index < text.length; index++) { + hash ^= text.charCodeAt(index); + hash = Math.imul(hash, 0x01000193); + } + return (hash >>> 0).toString(16).padStart(8, "0"); +} + +const PRELUDE_HASH = fnv1a(PRELUDE_SOURCE); + +/** mulberry32: a small, fast PRNG; one 32-bit draw per call. */ +function prng(seed: number): () => number { + let state = seed >>> 0; + return () => { + state = (state + 0x6d2b79f5) >>> 0; + let value = Math.imul(state ^ (state >>> 15), 1 | state); + value = (value + Math.imul(value ^ (value >>> 7), 61 | value)) ^ value; + return (value ^ (value >>> 14)) >>> 0; + }; +} + +/** The deterministic VM's clock: seed-derived while the context is created, then virtual time. */ +interface VirtualClock { + creating: boolean; + startedAt: number; + now: number; + seed: number; +} + +/** WASI clock and random functions for deterministic mode, on top of {@link discardOutput}. */ +function deterministicWasi(clock: VirtualClock) { + const random = prng(clock.seed ^ 0x9e3779b9); + return (memory: { readonly buffer: ArrayBufferLike }) => ({ + ...discardOutput(memory), + clock_time_get(clockId: number, _precision: bigint, resultPtr: number): number { + let nanoseconds: bigint; + // QuickJS seeds Math.random() from this reading in microseconds and turns 0 into 1, so seed s reads s + 1. + if (clock.creating) nanoseconds = clockId === 0 ? BigInt(clock.seed + 1) * 1000n : 0n; + else if (clockId === 0) nanoseconds = BigInt(Math.round(clock.now)) * 1_000_000n; + else nanoseconds = BigInt(Math.round(clock.now - clock.startedAt)) * 1_000_000n; + new DataView(memory.buffer).setBigUint64(resultPtr, nanoseconds, true); + return 0; + }, + random_get(bufPtr: number, bufLen: number): number { + const bytes = new Uint8Array(memory.buffer, bufPtr, bufLen); + for (let index = 0; index < bufLen; index += 4) { + const value = random(); + for (let byte = 0; byte < 4 && index + byte < bufLen; byte++) bytes[index + byte] = value >>> (byte * 8); + } + return 0; + }, + }); +} + /** * QuickJS writes engine diagnostics to fd 1 and 2, which the default shim * forwards to the host's stdout and stderr. That output belongs to the host @@ -80,30 +159,68 @@ function crashMessage(error: unknown): WorkerToHostMessage { export async function startVm(options: StartVmOptions): Promise { const { data, post } = options; + const maxStackSize = options.maxStackSize ?? MAX_STACK_SIZE; + const deterministic = data.deterministic; + const clock: VirtualClock | undefined = deterministic && { + creating: true, + startedAt: deterministic.startedAt, + now: deterministic.startedAt, + seed: deterministic.seed >>> 0, + }; const vm = await QuickJS.create({ wasm: options.wasm, memoryLimit: data.memoryLimitBytes, // Without a guard, deep recursion overflows the wasm stack and traps instead of throwing a // catchable RangeError. - maxStackSize: options.maxStackSize ?? MAX_STACK_SIZE, + maxStackSize, interruptHandler: options.shouldInterrupt, - wasi: discardOutput, + wasi: clock ? deterministicWasi(clock) : discardOutput, + ...(clock ? { timezoneOffset: 0, intrinsics: Intrinsics.ALL & ~Intrinsics.WEAK_REF } : {}), }); + if (clock) clock.creating = false; let disposed = false; + /** Calls the script has issued, tools and globals; reported with every `idle`. */ + let issued = 0; + /** Deterministic mode: the next result to apply, and results that arrived before it. */ + let nextSeq = 1; + /** The result being processed, 0 during the script's first run; reported with `done`. */ + let current = 0; + const early = new Map(); + const apply = (message: HostToWorkerMessage): void => { + vm.withScope(() => { + vm.callFunction( + settle, + api, + vm.newNumber(message.id), + message.ok ? vm.true : vm.false, + message.payload === undefined ? vm.undefined : vm.newString(message.payload), + message.properties === undefined ? vm.undefined : vm.newString(message.properties), + ); + }); + drain(); + }; const session: VmSession = { deliver(message) { if (disposed) return; try { - vm.withScope(() => { - vm.callFunction( - settle, - api, - vm.newNumber(message.id), - message.ok ? vm.true : vm.false, - message.payload === undefined ? vm.undefined : vm.newString(message.payload), - ); - }); - drain(); + if (!clock) { + apply(message); + return; + } + const { seq } = message; + if (seq === undefined || !Number.isInteger(seq) || seq < nextSeq || early.has(seq)) { + throw new Error(`Result ${message.id} has sequence number ${seq}, expected ${nextSeq} or later, once`); + } + early.set(seq, message); + for (let next = early.get(nextSeq); next !== undefined; next = early.get(nextSeq)) { + early.delete(nextSeq); + // Never backwards: a resumed host's clock may be behind the one that started the run. + if (next.now !== undefined && Number.isFinite(next.now)) clock.now = Math.max(clock.now, next.now); + current = nextSeq; + apply(next); + post({ type: "idle", seq: nextSeq, issued }); + nextSeq++; + } } catch (error) { post(crashMessage(error)); } @@ -138,6 +255,7 @@ export async function startVm(options: StartVmOptions): Promise { switch (kind.toString()) { case "call": case "global": + issued++; post({ type: "call", id: a.toNumber(), @@ -157,18 +275,22 @@ export async function startVm(options: StartVmOptions): Promise { : { type: "text", text: b.toString() }, }); break; - case "done": + case "done": { + // The host stops the VM on `done`, so the `idle` of this drain never arrives: report its numbers here. + const during = clock ? { during: { seq: current, issued } } : {}; if (a.toBoolean()) { post({ type: "done", ok: true, value: b === undefined || b.isUndefined ? undefined : b.toString(), writes: c.toString(), + ...during, }); } else { - post({ type: "done", ok: false, error: b.toString() }); + post({ type: "done", ok: false, error: b.toString(), ...during }); } break; + } } return vm.undefined; }); @@ -189,6 +311,14 @@ export async function startVm(options: StartVmOptions): Promise { settle = api.getProp("settle"); const run = api.getProp("run"); stalled = api.getProp("stalled"); + if (clock) { + const versions = vm.versions; + const budget = options.interruptBudget ?? "none"; + post({ + type: "started", + fingerprint: `quickjs-wasi ${versions["quickjs-wasi"]}, quickjs ${versions.quickjs}, prelude ${PRELUDE_HASH}, deterministic ${DETERMINISTIC_VERSION}, stack ${maxStackSize}, heap ${data.memoryLimitBytes ?? "none"}, budget ${budget}`, + }); + } // The prefix shares the first line with the script so reported line numbers // match the script as written. @@ -203,6 +333,7 @@ export async function startVm(options: StartVmOptions): Promise { vm.callFunction(run, api, fn).dispose(); fn.dispose(); drain(); + if (clock) post({ type: "idle", seq: 0, issued }); } catch (error) { post(crashMessage(error)); } diff --git a/packages/codemode/src/types.ts b/packages/codemode/src/types.ts index de0814391..9290311b7 100644 --- a/packages/codemode/src/types.ts +++ b/packages/codemode/src/types.ts @@ -86,7 +86,11 @@ export interface CodemodeStoreWrites { delete: string[]; } -/** `output` is kept for failed executions too, up to the failure. `exit()` completes with `value: undefined`. */ +/** + * `output` is kept for failed executions too, up to the failure. `exit()` completes with `value: undefined`. + * `during`: in a {@link CodemodeSandboxBase.start} run that the script itself ended (it returned, threw, or exited), + * the result being processed when it did. + */ export type CodemodeResult = | { ok: true; @@ -94,8 +98,15 @@ export type CodemodeResult = output: CodemodeOutputItem[]; calls: CodemodeCall[]; storeWrites: CodemodeStoreWrites; + during?: CodemodeDuring; } - | { ok: false; error: CodemodeError; output: CodemodeOutputItem[]; calls: CodemodeCall[] }; + | { + ok: false; + error: CodemodeError; + output: CodemodeOutputItem[]; + calls: CodemodeCall[]; + during?: CodemodeDuring; + }; /** Options every sandbox accepts, wherever its VM runs. */ export interface CodemodeSandboxBaseOptions { @@ -137,6 +148,81 @@ export interface CodemodeSandboxOptions extends CodemodeSandboxBaseOptions { workerUrl?: string | URL; } +/** A call the script made in a {@link CodemodeSandboxBase.start} run, answered with `CodemodeRun.deliver()`. */ +export interface CodemodeHostCall { + /** The VM's own number for the call, `1`, `2`, ... in the order the script made its calls (tools and globals). */ + id: number; + target: "tool" | "global"; + name: string; + /** What the script passed, after a JSON round trip; for spread globals the argument list. */ + args: unknown; +} + +/** A tool as a {@link CodemodeSandboxBase.start} run's script sees it: `tools.` and its `ALL_TOOLS` entry. */ +export interface CodemodeDeclaredTool { + name: string; + /** + * The identifier the script uses, `tools.`. Default: `toCodemodeIdentifier(name)`. A host that replays a run + * passes the one the run started with, so a changed mapping cannot rename a tool under a resumed script. + */ + jsName?: string; + description?: string; +} + +/** A global as a {@link CodemodeSandboxBase.start} run's script sees it; the same names as `globals` allow. */ +export interface CodemodeDeclaredGlobal { + name: string; + spread?: boolean; +} + +/** + * A call's answer. A rejection becomes an `Error` with `message` in the script, carrying the own properties of + * `properties` (JSON values; `name`, `message`, and `stack` are ignored), for example `{ structuredOutput }`. + */ +export type CodemodeOutcome = + | { ok: true; value: unknown } + | { ok: false; message: string; properties?: Record }; + +/** In a {@link CodemodeSandboxBase.start} run: the result the VM was processing when the script finished, 0 for its first run, and the calls issued by then. */ +export interface CodemodeDuring { + seq: number; + issued: number; +} + +export interface CodemodeRunOptions { + /** The VM clock's start, in ms since the epoch: `Date.now()` until the first result arrives. */ + startedAt: number; + /** Seeds `Math.random()` and the VM's other randomness. A 32-bit unsigned integer. */ + seed: number; + tools?: CodemodeDeclaredTool[]; + globals?: CodemodeDeclaredGlobal[]; + store?: Readonly>; + signal?: AbortSignal; + /** Deadline of this run in ms. Default: none (not the sandbox's default). */ + timeoutMs?: number; + /** The VM's fingerprint, before the script runs and before any of its calls. */ + onStarted?(fingerprint: string): void; + /** The script made a call. Answer it with `CodemodeRun.deliver()`, now or later. */ + onCall(call: CodemodeHostCall): void; + /** + * The VM processed result `seq` (`0`: the script's first run, before any result) and waits again, having issued + * `issued` calls in total. + */ + onIdle?(seq: number, issued: number): void; +} + +/** A running {@link CodemodeSandboxBase.start} execution. */ +export interface CodemodeRun { + readonly result: Promise; + /** + * Answer call `id` as result number `seq`. Results are numbered 1, 2, ... without gaps, and the VM applies them + * strictly in that order, whatever order they arrive in: after a gap it waits for the missing number. While it + * processes this one, its clock reads `now` (ms since the epoch), or the previous value if `now` is earlier. + * Throws for a call that is not waiting for an answer, a `seq` used before, or a `now` that is not finite. + */ + deliver(seq: number, now: number, id: number, outcome: CodemodeOutcome): void; +} + export interface CodemodeExecuteOptions { signal?: AbortSignal; /** Overrides the sandbox default for this execution. */ diff --git a/packages/codemode/test/deterministic.test.ts b/packages/codemode/test/deterministic.test.ts new file mode 100644 index 000000000..a383799a6 --- /dev/null +++ b/packages/codemode/test/deterministic.test.ts @@ -0,0 +1,400 @@ +import { afterEach, describe, expect, it } from "vitest"; +import { + type CodemodeHostCall, + type CodemodeOutcome, + type CodemodeRemoteExchange, + type CodemodeRun, + type CodemodeRunOptions, + CodemodeSandbox, + type CodemodeSandboxBase, + type CodemodeSandboxBaseOptions, + loadQuickJSWasm, + RemoteCodemodeSandbox, + serveCodemodeRemote, +} from "../src/index.ts"; + +/** + * `start()` runs a script deterministically with the caller answering its calls, so a durable host can replay it + * after a restart. Every test runs on the worker sandbox (Node) and the remote sandbox (Cloudflare), whose transport + * clones every message as Workers RPC would. + */ +const BUDGET = 50_000; +/** `delayFirst` holds the reply to the first result back, so later results reach the VM before it. */ +function remote(options: CodemodeSandboxBaseOptions, delayFirst = false): RemoteCodemodeSandbox { + return new RemoteCodemodeSandbox({ + ...options, + remote: () => ({ + execute: (request, exchange) => { + const wrapped: CodemodeRemoteExchange = async (messages) => { + const reply = structuredClone(await exchange(structuredClone(messages))); + if (delayFirst && reply?.type === "result" && reply.seq === 1) { + await new Promise((resolve) => setTimeout(resolve, 30)); + } + return reply; + }; + return serveCodemodeRemote(structuredClone(request), wrapped, { + wasm: loadQuickJSWasm(), + maxInterruptBudget: BUDGET, + }); + }, + }), + }); +} +const BACKENDS: Record CodemodeSandboxBase> = { + worker: (options = {}) => new CodemodeSandbox(options), + remote: (options = {}) => remote(options), +}; + +const STARTED_AT = Date.UTC(2026, 0, 15, 10, 0, 0); + +type Recorded = { + run: CodemodeRun; + calls: CodemodeHostCall[]; + idles: [seq: number, issued: number][]; + events: string[]; + fingerprint: () => string | undefined; +}; + +/** + * Start `code` with tools `echo` and `wait`. Every call but `wait` is answered at once, by default with its argument, + * as result number 1, 2, ... with the clock at `STARTED_AT + 1000 * seq`; `wait` is left for the test to answer. + */ +function start( + sandbox: CodemodeSandboxBase, + code: string, + options: Partial & { answer?: (call: CodemodeHostCall) => CodemodeOutcome } = {}, +): Recorded { + const calls: CodemodeHostCall[] = []; + const idles: [number, number][] = []; + const events: string[] = []; + let fingerprint: string | undefined; + let seq = 0; + const answer = options.answer ?? ((call: CodemodeHostCall) => ({ ok: true, value: call.args })); + const run: CodemodeRun = sandbox.start(code, { + startedAt: STARTED_AT, + seed: 1, + tools: [{ name: "echo" }, { name: "wait" }], + timeoutMs: 20_000, + ...options, + onStarted: (value) => { + fingerprint = value; + events.push("started"); + }, + onCall: (call) => { + calls.push(call); + events.push(`call ${call.id}`); + if (call.name === "wait") return; + queueMicrotask(() => { + seq++; + run.deliver(seq, STARTED_AT + 1000 * seq, call.id, answer(call)); + }); + }, + onIdle: (seq, issued) => { + idles.push([seq, issued]); + events.push(`idle ${seq}`); + }, + }); + return { run, calls, idles, events, fingerprint: () => fingerprint }; +} + +const savedTimezone = process.env.TZ; +afterEach(() => { + if (savedTimezone === undefined) delete process.env.TZ; + else process.env.TZ = savedTimezone; +}); + +for (const [backend, create] of Object.entries(BACKENDS)) { + describe(`deterministic runs (${backend})`, () => { + it("makes the same calls and output for the same start, seed, and results", async () => { + const code = ` + const values = [Date.now(), performance.now(), Math.random(), Math.random(), new Date().toISOString()]; + const first = await tools.echo(values); + text(JSON.stringify([Date.now(), performance.now(), Math.random()])); + return await tools.echo(first.concat(Math.random())); + `; + const sandbox = create(); + const one = start(sandbox, code); + const two = start(sandbox, code); + const [a, b] = await Promise.all([one.run.result, two.run.result]); + // Call durations are wall-clock time; everything else is the same. + const timeless = (result: typeof a) => ({ + ...result, + calls: result.calls.map(({ name, status }) => ({ name, status })), + }); + expect(timeless(a)).toEqual(timeless(b)); + expect(one.calls).toEqual(two.calls); + // The clock: startedAt, then the time of the result being processed; performance.now() counts from startedAt. + const [now, elapsed, random] = one.calls[0]!.args as number[]; + expect([now, elapsed]).toEqual([STARTED_AT, 0]); + expect((one.calls[0]!.args as unknown[])[4]).toBe("2026-01-15T10:00:00.000Z"); + const printed = JSON.parse(a.ok ? (a.output[0] as { text: string }).text : "[]") as number[]; + expect(printed.slice(0, 2)).toEqual([STARTED_AT + 1000, 1000]); + // Pinned: a change here changes what replayed scripts see, so it must change the fingerprint too. + expect(random).toBe(PINNED_FIRST_RANDOM); + + // The seed decides the random values, not the start time. + const later = start(sandbox, code, { startedAt: STARTED_AT + 5000 }); + const reseeded = start(sandbox, code, { seed: 2 }); + await Promise.all([later.run.result, reseeded.run.result]); + expect((later.calls[0]!.args as number[])[2]).toBe(random); + expect((reseeded.calls[0]!.args as number[])[2]).not.toBe(random); + // Seed 0 has its own values: QuickJS turns a zero state into 1, which must not make it seed 1's. + const zero = start(sandbox, "return [Math.random(), Math.random()]", { seed: 0 }); + const zeroResult = await zero.run.result; + expect(zeroResult).toMatchObject({ ok: true, value: PINNED_SEED_ZERO }); + expect((zeroResult.ok ? zeroResult.value : []) as number[]).not.toContain(random); + }); + + it("runs in UTC whatever the host's timezone, and without WeakRef or FinalizationRegistry", async () => { + process.env.TZ = "Asia/Kolkata"; + const code = + "return [new Date(0).getHours(), new Date(0).getTimezoneOffset(), typeof WeakRef, typeof FinalizationRegistry]"; + const sandbox = create(); + const { run } = start(sandbox, code); + expect(await run.result).toMatchObject({ ok: true, value: [0, 0, "undefined", "undefined"] }); + // execute() is unchanged: the host's timezone and every intrinsic. + expect(await sandbox.execute(code)).toMatchObject({ ok: true, value: [5, -330, "function", "function"] }); + }); + + it("applies results in seq order, whatever order they are delivered in", async () => { + const sandbox = create(); + const recorded = start( + sandbox, + `const order = []; + await Promise.all([tools.wait("a").then(() => order.push("a")), tools.wait("b").then(() => order.push("b"))]); + return order;`, + ); + await waitFor(() => recorded.calls.length === 2 && recorded.idles.length === 1); + const [a, b] = recorded.calls; + // b is result 1 and a result 2, but a arrives first. + recorded.run.deliver(2, STARTED_AT + 2000, a!.id, { ok: true, value: null }); + recorded.run.deliver(1, STARTED_AT + 1000, b!.id, { ok: true, value: null }); + expect(await recorded.run.result).toMatchObject({ ok: true, value: ["b", "a"] }); + // No idle after the last result: the script finished in that drain, and the host stopped the VM. + expect(recorded.idles).toEqual([ + [0, 2], + [1, 2], + ]); + }); + + it("reports the fingerprint before any call and the calls issued after each result", async () => { + const sandbox = create({ memoryLimitBytes: 64 * 1024 * 1024 }); + const recorded = start(sandbox, "await tools.echo(1); await Promise.all([tools.echo(2), tools.echo(3)]);"); + expect(await recorded.run.result).toMatchObject({ ok: true }); + expect(recorded.events.slice(0, 3)).toEqual(["started", "call 1", "idle 0"]); + expect(recorded.idles).toEqual([ + [0, 1], + [1, 3], + [2, 3], + ]); + expect(recorded.calls.map((call) => [call.id, call.target, call.name, call.args])).toEqual([ + [1, "tool", "echo", 1], + [2, "tool", "echo", 2], + [3, "tool", "echo", 3], + ]); + const fingerprint = recorded.fingerprint()!; + expect(fingerprint).toMatch( + /^quickjs-wasi \S+, quickjs \S+, prelude [0-9a-f]{8}, deterministic \d+, stack \d+, heap 67108864, budget \S+$/, + ); + if (backend === "remote") expect(fingerprint).toContain(`budget ${BUDGET}`); + else expect(fingerprint).toContain("budget none"); + + const same = start(sandbox, "return 1"); + const other = start(create({ memoryLimitBytes: 32 * 1024 * 1024 }), "return 1"); + await Promise.all([same.run.result, other.run.result]); + expect(same.fingerprint()).toBe(fingerprint); + expect(other.fingerprint()).not.toBe(fingerprint); + }); + + it("numbers calls to globals with tools, and answers errors as rejections", async () => { + const sandbox = create(); + const recorded = start( + sandbox, + `const found = await search("x", 2); + try { await tools.echo("fail"); } catch (error) { text(error.message); } + return found;`, + { + globals: [{ name: "search", spread: true }], + answer: (call) => + call.args === "fail" + ? { ok: false, message: "no" } + : { ok: true, value: call.name === "search" ? ["found"] : call.args }, + }, + ); + const result = await recorded.run.result; + expect(recorded.calls[0]).toEqual({ id: 1, target: "global", name: "search", args: ["x", 2] }); + expect(result).toMatchObject({ ok: true, value: ["found"], output: [{ type: "text", text: "no" }] }); + expect(recorded.calls[1]).toMatchObject({ id: 2, target: "tool", name: "echo" }); + // Calls to globals count as issued calls too. + expect(recorded.idles.slice(0, 2)).toEqual([ + [0, 1], + [1, 2], + ]); + expect(result.calls).toMatchObject([{ name: "echo", status: "error" }]); + }); + + it("keeps the clock monotonic and rejects a clock value that is not finite", async () => { + const sandbox = create(); + const recorded = start( + sandbox, + "const before = Date.now(); await tools.wait(1); return [before, Date.now(), performance.now()]", + ); + await waitFor(() => recorded.calls.length === 1); + expect(() => recorded.run.deliver(1, Number.NaN, 1, { ok: true, value: null })).toThrow("not a finite number"); + // A resumed host whose clock is behind the run's start. + recorded.run.deliver(1, STARTED_AT - 2000, 1, { ok: true, value: null }); + expect(await recorded.run.result).toMatchObject({ ok: true, value: [STARTED_AT, STARTED_AT, 0] }); + }); + + it("reports the result the VM was processing when the script ended", async () => { + const sandbox = create(); + const returned = start(sandbox, "await tools.echo(1); return await tools.echo(2)"); + const thrown = start( + sandbox, + "await tools.echo(1); await Promise.all([tools.echo(2), tools.echo(3)]); throw new Error('x')", + ); + const first = start(sandbox, "return 1"); + expect(await returned.run.result).toMatchObject({ ok: true, value: 2, during: { seq: 2, issued: 2 } }); + expect(await thrown.run.result).toMatchObject({ ok: false, during: { seq: 3, issued: 3 } }); + expect(await first.run.result).toMatchObject({ ok: true, during: { seq: 0, issued: 0 } }); + expect(await sandbox.execute("return 1")).not.toHaveProperty("during"); + }); + + it("leaves a call waiting when its answer cannot be serialized", async () => { + const sandbox = create(); + const recorded = start(sandbox, "return await tools.wait(1)"); + await waitFor(() => recorded.calls.length === 1); + expect(() => recorded.run.deliver(1, STARTED_AT, 1, { ok: true, value: 1n })).toThrow(); + expect(() => + recorded.run.deliver(1, STARTED_AT, 1, { ok: false, message: "x", properties: { big: 1n } }), + ).toThrow(); + recorded.run.deliver(1, STARTED_AT, 1, { ok: true, value: "fine" }); + expect(await recorded.run.result).toMatchObject({ ok: true, value: "fine" }); + }); + + it("rejects with an Error carrying the answer's properties", async () => { + const sandbox = create(); + const recorded = start( + sandbox, + "try { await tools.echo(1); } catch (e) { return [e instanceof Error, e.message, e.name, e.structuredOutput]; }", + { + answer: () => ({ + ok: false, + message: "denied", + properties: { structuredOutput: { code: 7 }, name: "Ignored", message: "ignored" }, + }), + }, + ); + expect(await recorded.run.result).toMatchObject({ ok: true, value: [true, "denied", "Error", { code: 7 }] }); + }); + + it("rejects rejection properties that are not an object, and tools sharing a script name", async () => { + const sandbox = create(); + const recorded = start(sandbox, "return await tools.wait(1)"); + await waitFor(() => recorded.calls.length === 1); + for (const properties of [null, [1]] as unknown as Record[]) { + expect(() => recorded.run.deliver(1, STARTED_AT, 1, { ok: false, message: "x", properties })).toThrow( + "not an object", + ); + } + recorded.run.deliver(1, STARTED_AT, 1, { ok: true, value: 1 }); + expect(await recorded.run.result).toMatchObject({ ok: true, value: 1 }); + expect(() => + sandbox.start("", { + startedAt: STARTED_AT, + seed: 1, + tools: [{ name: "a-b" }, { name: "a_b" }], + onCall: () => {}, + }), + ).toThrow("script names must be unique"); + }); + + it("uses the script names the run passes", async () => { + const sandbox = create(); + const recorded = start( + sandbox, + "return [typeof tools.renamed, Object.keys(tools), ALL_TOOLS.map((tool) => tool.name)]", + { + tools: [{ name: "my-tool", jsName: "renamed" }], + }, + ); + expect(await recorded.run.result).toMatchObject({ + ok: true, + value: ["function", ["renamed", "my-tool"], ["renamed"]], + }); + }); + + it("ends with an error the script cannot catch when the caller aborts", async () => { + const sandbox = create(); + const controller = new AbortController(); + const recorded = start( + sandbox, + `try { await tools.wait(1); } catch { text("caught"); } + text("after");`, + { signal: controller.signal }, + ); + await waitFor(() => recorded.calls.length === 1); + controller.abort(new Error("diverged")); + const result = await recorded.run.result; + expect(result).toMatchObject({ ok: false, error: { kind: "aborted", message: "diverged" }, output: [] }); + expect(() => recorded.run.deliver(1, STARTED_AT, 1, { ok: true, value: null })).not.toThrow(); + }); + + it("rejects a call that is not waiting and a result number used twice", async () => { + const sandbox = create(); + const recorded = start(sandbox, "await Promise.all([tools.wait(1), tools.wait(2)])"); + await waitFor(() => recorded.calls.length === 2); + const { run } = recorded; + expect(() => run.deliver(1, STARTED_AT, 99, { ok: true, value: null })).toThrow("Call 99 is not waiting"); + run.deliver(1, STARTED_AT, 1, { ok: true, value: null }); + expect(() => run.deliver(1, STARTED_AT, 2, { ok: true, value: null })).toThrow("used once"); + expect(() => run.deliver(0, STARTED_AT, 2, { ok: true, value: null })).toThrow("used once"); + run.deliver(2, STARTED_AT, 2, { ok: true, value: null }); + expect(await run.result).toMatchObject({ ok: true }); + }); + }); +} + +describe("deterministic runs (remote transport)", () => { + it("applies results in seq order when the transport reorders replies sent in order", async () => { + const sandbox = remote({}, true); + const recorded = start( + sandbox, + `const order = []; + await Promise.all([tools.wait("a").then(() => order.push("a")), tools.wait("b").then(() => order.push("b"))]); + return order;`, + ); + await waitFor(() => recorded.calls.length === 2); + const [a, b] = recorded.calls; + recorded.run.deliver(1, STARTED_AT + 1000, a!.id, { ok: true, value: null }); + recorded.run.deliver(2, STARTED_AT + 2000, b!.id, { ok: true, value: null }); + expect(await recorded.run.result).toMatchObject({ ok: true, value: ["a", "b"] }); + }); +}); + +describe("execute() is not deterministic", () => { + it("reads the real clock and real randomness", async () => { + const sandbox = new CodemodeSandbox(); + const before = Date.now(); + const first = await sandbox.execute("return [Date.now(), Math.random()]"); + const second = await sandbox.execute("return [Date.now(), Math.random()]"); + const [now, random] = (first.ok ? first.value : []) as number[]; + expect(now).toBeGreaterThanOrEqual(before); + expect(now).toBeLessThanOrEqual(Date.now()); + expect((second.ok ? second.value : []) as number[]).not.toContain(random); + }); +}); + +async function waitFor(check: () => boolean): Promise { + const deadline = Date.now() + 5000; + while (!check()) { + if (Date.now() > deadline) throw new Error("Condition was not reached"); + await new Promise((resolve) => setTimeout(resolve, 5)); + } +} + +/** The first two `Math.random()` values with seed 0. */ +const PINNED_SEED_ZERO = [0.6711372530266764, 0.7258461452833667]; + +/** The first `Math.random()` with seed 1. */ +const PINNED_FIRST_RANDOM = 0.34227450605335275;