From 8b0caefc2260a28d2cec0a7ce3daef1d3de24776 Mon Sep 17 00:00:00 2001 From: Mario Zechner Date: Mon, 26 Oct 2026 20:44:20 +0000 Subject: [PATCH] feat(durable,chord): publish commits at adoption, address-following watches, synchronous observers Durable publishes a stored commit inside Storage.commit's adopt: the telemetry events, its own observers (document watches, conversation views, task graph, scheduler, waits), and a dispatch of host subscribeCommits listeners one microtask later, before every wait and abort the commit triggers. A commit the storage rejects after adoption is published before the Session fails; a throwing telemetry adapter is reported as host_error "telemetry" instead of rejecting a durable commit. Chord's watch listeners take one change object, { value, ops, retired } ({ value } for native watches); retirement is a flag, not an end. Durable's document watches follow their address, and a task document's watch ends after its retirement call. New synchronous observers for adapters: observeDoc, observeConversation, observeTaskGraph; the watches are built on them. The env file watcher's change type is renamed FileChange. --- packages/chord/CHANGELOG.md | 8 +- packages/chord/README.md | 38 +- packages/chord/src/index.ts | 2 + packages/chord/src/watch.ts | 226 +++++--- packages/chord/test/watch.test.ts | 424 +++++++++++--- packages/durable/CHANGELOG.md | 17 +- packages/durable/README.md | 16 +- packages/durable/docs/spec.md | 293 ++++++++-- packages/durable/src/env/index.ts | 6 +- packages/durable/src/env/node-watch.ts | 10 +- packages/durable/src/env/node.ts | 4 +- packages/durable/src/harness/events.ts | 11 +- packages/durable/src/harness/harness.ts | 33 +- packages/durable/src/harness/scheduler.ts | 29 +- packages/durable/src/harness/task-graph.ts | 47 +- packages/durable/src/harness/types.ts | 33 +- packages/durable/src/harness/view.ts | 40 +- packages/durable/src/index.ts | 4 +- packages/durable/src/session/session.ts | 533 ++++++++++++++---- .../durable/src/testing/env-conformance.ts | 10 +- packages/durable/src/types.ts | 99 +++- .../durable/test/env-node-conformance.test.ts | 4 +- packages/durable/test/examples/04-watches.ts | 2 +- .../durable/test/examples/15-real-model.ts | 2 +- packages/durable/test/examples/18-json.ts | 4 +- .../durable/test/examples/20-late-join.ts | 5 +- .../test/examples/21-subagent-foreground.ts | 2 +- .../test/examples/22-subagent-background.ts | 3 +- .../durable/test/harness-compaction.test.ts | 12 +- packages/durable/test/harness-events.test.ts | 8 +- .../test/harness-generation-recovery.test.ts | 4 +- .../test/harness-input-entries.test.ts | 2 +- .../test/harness-nested-tools-restart.test.ts | 2 +- .../durable/test/harness-nested-tools.test.ts | 12 +- .../durable/test/harness-observers.test.ts | 459 +++++++++++++++ .../test/harness-storage-failure.test.ts | 2 +- .../test/harness-structured-output.test.ts | 2 +- .../durable/test/harness-structured.test.ts | 4 +- .../durable/test/harness-submissions.test.ts | 18 +- .../durable/test/harness-task-graph.test.ts | 33 +- packages/durable/test/harness-tasks.test.ts | 276 ++++++++- .../durable/test/harness-telemetry.test.ts | 92 ++- packages/durable/test/harness-view.test.ts | 4 +- .../durable/test/session-adoption.test.ts | 268 ++++++++- .../session-checkpoints-migrations.test.ts | 6 +- .../test/session-deferred-commits.test.ts | 4 +- .../test/session-document-retention.test.ts | 25 +- .../durable/test/session-observers.test.ts | 411 ++++++++++++++ packages/durable/test/session-support.ts | 50 +- packages/durable/test/session-watches.test.ts | 328 +++++++++-- packages/durable/test/task-support.ts | 2 + packages/env/src/remote-env.ts | 4 +- packages/env/src/watch.ts | 10 +- 53 files changed, 3389 insertions(+), 554 deletions(-) create mode 100644 packages/durable/test/harness-observers.test.ts create mode 100644 packages/durable/test/session-observers.test.ts diff --git a/packages/chord/CHANGELOG.md b/packages/chord/CHANGELOG.md index e8b74b5e9..584aaddd0 100644 --- a/packages/chord/CHANGELOG.md +++ b/packages/chord/CHANGELOG.md @@ -5,14 +5,16 @@ ### Breaking Changes - `traceIn(options, fn, context)` takes the context last (was `traceIn(context, options, fn)`), like the rest of the API. +- Watch listeners take one change object before the context: `WatchHandle.start((change, context) => …)` with `change: WatchChange` = `{ value, ops, retired }`, and `NativeWatchHandle.start(({ value }, context) => …)` with `NativeWatchChange`. A change is plain data, those fields only; the frame's context is the second argument. `Watch.advance(change, context)` takes the same object, and a frame delivered alone, with nothing folded into it, reaches the listener as the object the source passed. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`, `start(async (value, ctx) => …)` on a native watch `start(async ({ value }, ctx) => …)`, and `advance(value, ops, ctx)` becomes `advance({ value, ops, retired: false }, ctx)`. +- Retirement is a flag, not an end: `WatchEnd` loses `retired`, and `Watch.retire(context)` is removed. A source reports a retirement with `advance({ value, ops, retired: true }, context)`, whose value is what the address shows after it, and the watch goes on: the next call has `retired: true`, folded with every other pending frame, so a document created at the address afterwards arrives in the same or a later call as `[["r", value]]`. `endAfterDelivery()` tells a watch its source sends nothing more: it takes no more frames, delivers what is pending, and ends `ended` after its last call returned, at once when nothing is pending and no call runs; it stays attached until then, so the source's `end()` (such as at its close), `stop()`, cancellation, or a throwing listener still end it first. Migration: a listener that waited for `closed` to give `retired` or tested for a `null` value tests `change.retired`; a source that called `retire(ctx)` calls `advance({ value, ops, retired: true }, ctx)`, then `endAfterDelivery()` when its owner ends with the retirement. ### Added - `NativeWatchable`: an in-process value that changes and can be watched (`value`, `watch(context)` returning a `NativeWatchHandle`), the in-process counterpart of `Watchable`. -- `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`. `WatchEnd` reasons are `stopped`, `cancelled`, `retired`, `ended`, `listener_error`, and `failed`. -- `Watch`, the one watch handle implementation, moved from Durable: a source creates one per watch and feeds it with `advance(value, ops, context)`, `retire(context)` (last frame `[["r", null]]`, then `retired`), `end()` or `end({ error })`, and `observeCancellation(signal)`. Frames that arrive while the listener runs (or before `start`) are delivered in one call with the newest value, their ops folded by `PendingOps` and the newest frame's context; past 10,000 pending ops after folding, the call gets one root replacement `[["r", newest]]` instead (replaces the collapse after 100 undelivered frames); termination drops the listener and every reference into the source. +- `WatchHandle` and `WatchEnd`, the one watch API, moved from Durable: a serialized observation with `value`, `start(listener)`, `stop()`, and `closed`; the listener gets a `WatchChange` (`{ value, ops, retired }`) and the context. `WatchEnd` reasons are `stopped`, `cancelled`, `ended`, `listener_error`, and `failed`. +- `Watch`, the one watch handle implementation, moved from Durable: a source creates one per watch and feeds it with `advance(change, context)` (`retired: true` for a retirement), `end()` or `end({ error })`, `endAfterDelivery()`, and `observeCancellation(signal)`. Frames that arrive while the listener runs (or before `start`) are delivered in one call with the newest value, their ops folded by `PendingOps` and the newest frame's context; past 10,000 pending ops after folding, the call gets one root replacement `[["r", newest]]` instead (replaces the collapse after 100 undelivered frames); termination drops the listener and every reference into the source. - `Watchable`: a value that changes and can be watched (`value`, `watch(context)` returning a `WatchHandle`), the common interface of documents, derives and other observable values. -- `NativeWatchHandle` and its implementation `NativeWatch`: the same lifecycle for in-process values, with listener `(value, context)` and no ops: one call per value in order, and past 100 pending values one call with the value of the optional `replace()` or the newest value. +- `NativeWatchHandle` and its implementation `NativeWatch`: the same lifecycle for in-process values, with listener `({ value }, context)` (`NativeWatchChange`) and no ops: one call per value in order, and past 100 pending values one call with the value of the optional `replace()` or the newest value. - `PendingOps` and `PendingTake` in `@earendil-works/chord/delta`: the pending ops of one slow consumer, kept raw and folded while they wait (appends to one path join, a primitive set replaces earlier leaf ops on its path, `r` drops everything before it); `size` counts the pending ops after folding; `take()` empties the queue and hands over the folded ops, or `replace` past 10,000 pending ops. Received batches and op tuples are never mutated. - `ownedSlice(text, start, end?)` in `@earendil-works/chord/delta`: a slice of a string that does not keep the whole string alive ([#9265](https://github.com/earendil-works/pi/issues/9265)) - `appendText(draft, key, text)` in `@earendil-works/chord/delta`: append to a string in a draft at O(text) cost, recorded as an append op without comparing the old and new strings (a plain `draft[key] += text` makes `prepare()` read the whole string, O(length) per change); `StringKeys` names the keys it accepts ([#9265](https://github.com/earendil-works/pi/issues/9265)) diff --git a/packages/chord/README.md b/packages/chord/README.md index 4239568bc..376b3625d 100644 --- a/packages/chord/README.md +++ b/packages/chord/README.md @@ -14,7 +14,7 @@ Contracts are runtime values described with [TypeBox](https://github.com/sinclai | Import | Contents | Runs in | |--------|----------|---------| -| `@earendil-works/chord` | Contracts (`local`, `remote`, `method`, `opaque`), declarations (through the factory's `self`: `state`, `derive`, `resource`, `on`, `task`), hosts (`createHost`, `defineFacet`, `defineRegistry`), links (`memoryLinkPair`, `ipcLink`, `webSocketLink`, `acceptWebSockets`), errors (`ChordError`, `isChordError`, `forbidden`, `invalid`). Also the shared base: `Context`, `JsonValue`, `JsonRepresentation`, `Draft`, `copyJson`, `isJsonValue`, `replicatedState` with its types, and the watch types `WatchHandle`, `NativeWatchHandle`, and `WatchEnd` with their implementations `Watch` and `NativeWatch`. | Node and browsers | +| `@earendil-works/chord` | Contracts (`local`, `remote`, `method`, `opaque`), declarations (through the factory's `self`: `state`, `derive`, `resource`, `on`, `task`), hosts (`createHost`, `defineFacet`, `defineRegistry`), links (`memoryLinkPair`, `ipcLink`, `webSocketLink`, `acceptWebSockets`), errors (`ChordError`, `isChordError`, `forbidden`, `invalid`). Also the shared base: `Context`, `JsonValue`, `JsonRepresentation`, `Draft`, `copyJson`, `isJsonValue`, `replicatedState` with its types, and the watch types `WatchHandle`, `NativeWatchHandle`, `WatchChange`, `NativeWatchChange`, and `WatchEnd` with their implementations `Watch` and `NativeWatch`. | Node and browsers | | `@earendil-works/chord/node` | Child host processes: `createHostManager`, `forkDriver`, `childControl`, `processLock`; the facet module loader `nodeLoader`. | Node | | `@earendil-works/chord/context` | Explicit cancellation, telemetry, and invocation values: `BACKGROUND_CONTEXT`, `withCancel`, `withAbortSignal`, `withoutAbortSignal`, `withTelemetry`, `traceIn`, `withContextValue`, `awaitWithContext`. | Node and browsers | | `@earendil-works/chord/delta` | The transactional JSON tracker, structural diff, immutable apply, and the wire encoder/decoder for operation batches. | Node and browsers | @@ -209,17 +209,26 @@ model; see [State](docs/guide.md#state) in the usage guide. `WatchHandle` is the one watch interface, implemented by watch sources such as Durable's documents, conversation views, and task graph. `watch.value` is the value at acquisition until `start(listener)` installs the sole asynchronous listener; each -later change arrives as `(value, ops, context)`, one call at a time, and `value` -advances to it. Changes made while the listener runs (or before `start`) arrive in -one call: the newest value, the ops of all of them folded by Delta's `PendingOps`, -and the newest change's context. Past 10,000 pending ops after folding the call -gets one root replacement `[["r", newest]]` instead. `stop()` is idempotent, drops -pending frames, and neither aborts nor awaits a running call; it and `closed` give -the `WatchEnd`: +later change arrives as `(change, context)`, one call at a time, with +`change: WatchChange` = `{ value, ops, retired }` (plain data; the context is +the second argument), and `value` advances to it. +Changes made while the listener runs (or before `start`) arrive in one call: the +newest value, the ops of all of them folded by Delta's `PendingOps`, and the newest +change's context. Past 10,000 pending ops after folding the call gets one root +replacement `[["r", newest]]` instead. + +`retired` says the document the watch follows was retired since the previous call. +It does not end the watch: the call carries what the address shows after the +retirement (the source decides; a Durable document keeps the last value the watch +was given, with no ops), and a document created there later arrives as +`[["r", value]]`, folded with what is pending, so one call can report a retirement +and carry the new value. + +`stop()` is idempotent, drops pending frames, and neither aborts nor awaits a +running call; it and `closed` give the `WatchEnd`: - `stopped`: `stop()` was called; - `cancelled`: the context the watch was acquired with was cancelled; -- `retired`: the source was retired, an explicit event after a last frame `[["r", null]]`; - `ended`: the source closed (its owner ended or closed); - `listener_error` with `error`: the listener threw or rejected; - `failed` with `error`: the source failed. @@ -227,9 +236,12 @@ the `WatchEnd`: An ended watch drops every reference into its source, so a stopped watch that is still referenced keeps only its last value alive. -Sources implement handles with `Watch`, fed through `advance(value, ops, context)`, -`retire(context)`, `end()` or `end({ error })`, and `observeCancellation(signal)`. +Sources implement handles with `Watch`, fed through `advance(change, context)` +(a `WatchChange`, `retired` set for a retirement), `end()` or `end({ error })` +(at once, dropping what is pending), `endAfterDelivery()` (take no more frames, +deliver what is pending, then end `ended` once the last call returned; the +source's `end()` still ends it first), and `observeCancellation(signal)`. `NativeWatch` implements `NativeWatchHandle`, the same lifecycle for -in-process values: the listener gets `(value, context)` without ops, one call per -value in order; past 100 pending values they collapse to one, the value of the +in-process values: the listener gets `({ value }, context)` without ops, one call +per value in order; past 100 pending values they collapse to one, the value of the optional `replace()` or the newest. diff --git a/packages/chord/src/index.ts b/packages/chord/src/index.ts index 8b8bd47d6..30ba50455 100644 --- a/packages/chord/src/index.ts +++ b/packages/chord/src/index.ts @@ -81,9 +81,11 @@ export type { Ctx, GetOptions, Readable, RefStatus, Shape, StateEvent } from "./ export { NativeWatch, type NativeWatchable, + type NativeWatchChange, type NativeWatchHandle, Watch, type Watchable, + type WatchChange, type WatchEnd, type WatchHandle, } from "./watch.ts"; diff --git a/packages/chord/src/watch.ts b/packages/chord/src/watch.ts index 76373de2f..74a7ef15b 100644 --- a/packages/chord/src/watch.ts +++ b/packages/chord/src/watch.ts @@ -8,16 +8,28 @@ import type { JsonValue } from "./json.ts"; * * - `stopped`: `stop()` was called. * - `cancelled`: the context the watch was acquired with was cancelled. - * - `retired`: the source was retired; its last frame was `[["r", null]]`. * - `ended`: the source closed (its owner ended or closed). * - `listener_error`: the listener threw or rejected with `error`. * - `failed`: the source failed with `error`. */ export type WatchEnd = - | { readonly reason: "stopped" | "cancelled" | "retired" | "ended" } + | { readonly reason: "stopped" | "cancelled" | "ended" } | { readonly reason: "listener_error"; readonly error: Error } | { readonly reason: "failed"; readonly error: unknown }; +/** + * One call of a JSON watch's listener, and one change a source hands its watch: the newest value, the ops that turn the + * previous value into it, and whether the document the watch follows was retired since the previous call. A retirement + * does not end the watch: `value` is then what the address shows after it (the source decides; a Durable document keeps + * the last value the watch was given, with no ops), and a document created there later arrives as `[["r", value]]`, + * possibly in the same call. Plain data: the frame's context is the listener's second argument, never part of the + * change. + */ +export type WatchChange = { readonly value: T; readonly ops: readonly Op[]; readonly retired: boolean }; + +/** One call of a native watch's listener: the new value. */ +export type NativeWatchChange = { readonly value: T }; + /** * Serialized observation of an immutable value: one listener call at a time; changes made meanwhile arrive folded into * the next call. @@ -26,7 +38,7 @@ export interface WatchHandle { /** Acquisition revision before start; latest delivered immutable revision afterward. */ readonly value: T; /** Install the sole asynchronous listener. Never invokes it inline. */ - start(listener: (value: T, ops: readonly Op[], context: Context) => Promise): void; + start(listener: (change: WatchChange, context: Context) => Promise): void; /** Idempotently stop future callbacks and return this watch's terminal result. */ stop(): Promise; /** Settle when the watch terminates; an already-running callback remains caller-owned. */ @@ -64,39 +76,32 @@ export interface NativeWatchHandle { /** Acquisition value before start; latest delivered value afterward. */ readonly value: T; /** Install the sole asynchronous listener. Never invokes it inline. */ - start(listener: (value: T, context: Context) => Promise): void; + start(listener: (change: NativeWatchChange, context: Context) => Promise): void; /** Idempotently stop future callbacks and return this watch's terminal result. */ stop(): Promise; /** Settle when the watch terminates; an already-running callback remains caller-owned. */ readonly closed: Promise; } -type WatchListener = (value: T, ops: readonly Op[], context: Context) => Promise; - -type NativeWatchListener = (value: T, context: Context) => Promise; - -/** One listener call: the value it delivers and the context of the newest frame it covers. */ -type Delivery = { - readonly value: T; - readonly context: Context; - /** The retirement: the watch ends `retired` once it is delivered. */ - readonly retires: boolean; -}; +type Listener = (change: C, context: Context) => Promise; /** * The lifecycle both handles share: serialized delivery and termination. A source feeds it and ends it explicitly; * termination drops every reference the watch holds into its source, so a stopped watch that is still referenced keeps - * nothing but its last value alive. Subclasses keep what is pending and say how a listener is called. + * nothing but its last value alive. Subclasses keep what is pending, each change apart from its frame's context, so the + * change a listener gets carries nothing but its data. */ -abstract class WatchLifecycle> { +abstract class WatchLifecycle { #detach: (() => void) | undefined; #report: ((error: unknown) => void) | undefined; readonly #closed = Promise.withResolvers(); #value: T; - #listener: L | undefined; + #listener: Listener | undefined; #started = false; #scheduled = false; #running = false; + /** Set by `endAfterDelivery()`: the source sends nothing more; the watch ends `ended` once it delivered the rest. */ + #ending = false; /** Set once, by `#terminate()`. */ #end: WatchEnd | undefined; #cancellationSignal: AbortSignal | undefined; @@ -116,7 +121,7 @@ abstract class WatchLifecycle> { return this.#closed.promise; } - start(listener: L): void { + start(listener: Listener): void { if (this.#started) throw new Error("Watch is already started"); if (this.#end !== undefined) throw new Error("Watch is stopped"); this.#started = true; @@ -144,21 +149,36 @@ abstract class WatchLifecycle> { this.#terminate({ reason: "cancelled" }); } - /** The source closed: end with `ended`, or with `failed` and the error when the source failed. */ + /** + * The source closed: end with `ended`, or with `failed` and the error when the source failed, at once, dropping what + * is pending. + */ end(failure?: { readonly error: unknown }): void { this.#terminate(failure === undefined ? { reason: "ended" } : { reason: "failed", error: failure.error }); } - /** Whether the watch still takes frames: it has not ended. */ - protected get open(): boolean { - return this.#end === undefined; + /** + * The source sends nothing more after its last frames, such as a retirement: take no more frames, deliver what is + * pending, then end `ended`, after the running call returns, if one runs. Until then the watch stays attached, so + * `end()`, `stop()`, cancellation, and a throwing listener still end it first; a watch never started keeps what is + * pending until one of them or `start()`. + */ + endAfterDelivery(): void { + if (this.#end !== undefined || this.#ending) return; + this.#ending = true; + if (!this.#running && this.pendingContext === undefined) this.#terminate({ reason: "ended" }); } - /** Take the next delivery, or undefined when nothing is pending. */ - protected abstract next(): D | undefined; + /** Whether the watch still takes frames: it has not ended, and its source has not ended. */ + protected get open(): boolean { + return this.#end === undefined && !this.#ending; + } - /** Call `listener` with one delivery. */ - protected abstract invoke(listener: L, delivery: D, context: Context): Promise; + /** The context of the newest frame the next call covers, or undefined when nothing is pending. */ + protected abstract get pendingContext(): Context | undefined; + + /** Take the next change; called only while something is pending. */ + protected abstract next(): C; /** Drop everything pending and every reference into the source; called once, on termination. */ protected abstract release(): void; @@ -177,7 +197,10 @@ abstract class WatchLifecycle> { }); } - /** Deliver until nothing is pending; what arrives during a call is taken after it returns. */ + /** + * Deliver until nothing is pending; what arrives during a call is taken after it returns. A watch whose source ended + * ends once it delivered the rest and its last call returned. + */ async #drain(): Promise { if (this.#running || this.#end !== undefined || !this.#started) return; this.#running = true; @@ -185,11 +208,15 @@ abstract class WatchLifecycle> { while (this.#end === undefined) { const listener = this.#listener; if (listener === undefined) break; - const delivery = this.next(); - if (delivery === undefined) break; - this.#value = delivery.value; + const frameContext = this.pendingContext; + if (frameContext === undefined) { + if (this.#ending) this.#terminate({ reason: "ended" }); + break; + } + const change = this.next(); + this.#value = change.value; try { - await this.invoke(listener, delivery, withoutAbortSignal(delivery.context)); + await listener(change, withoutAbortSignal(frameContext)); } catch (error) { if (this.#end === undefined) { this.#report?.(error); @@ -197,10 +224,6 @@ abstract class WatchLifecycle> { } break; } - if (delivery.retires) { - this.#terminate({ reason: "retired" }); - break; - } } } finally { this.#running = false; @@ -238,61 +261,67 @@ function toError(error: unknown): Error { } } -type WatchDelivery = Delivery & { readonly ops: readonly Op[] }; - -/** Canonical terminal frame of a retired source. */ -const RETIREMENT_OPERATIONS: readonly Op[] = [["r", null]]; - /** * The watch handle of a JSON source. Frames that arrive before `start()` or while the listener runs wait in one * `PendingOps` queue; the next call gets the newest value, the folded ops of every frame since the previous call (or - * `[["r", newest]]` after the queue overflowed) and the newest frame's context. `detach` unsubscribes it from its - * source on termination; `report` gets a listener's throw, which also ends the watch. + * `[["r", newest]]` after the queue overflowed), `retired` when any of them was a retirement, and the newest frame's + * context. A frame delivered alone is passed on as the source made it. `detach` unsubscribes it from its source on + * termination; `report` gets a listener's throw, which also ends the watch. */ -export class Watch extends WatchLifecycle, WatchDelivery> implements WatchHandle { +export class Watch extends WatchLifecycle> implements WatchHandle { readonly #ops = new PendingOps(); - /** The newest frame, while a call is pending. */ - #newest: { readonly value: T; readonly context: Context } | undefined; - #retiring = false; - - /** Queue the frame `ops`, which turn the previous value into `value`. */ - advance(value: T, ops: readonly Op[], context: Context): void { - if (!this.open || this.#retiring) return; - this.#ops.push(ops); - this.#newest = { value, context }; - this.pending(); - } + /** + * The source's change while it is the only frame pending: the call passes it on, with no allocation. A second frame + * drops it, so its ops stay only as long as `PendingOps` keeps them (folded, or dropped by an overflow). + */ + #lone: WatchChange | undefined; + /** The newest frame's value and context, while a call is pending; the context also marks that one is. */ + #newestValue: T | undefined; + #newestContext: Context | undefined; + /** A pending frame was a retirement. */ + #retired = false; /** - * The source retired: queue the last frame, `[["r", null]]` with value `null`, then end `retired`. Typed for watches - * whose values include `null`. + * Queue the frame `change`: its ops turn the previous value into its value, and `retired` says the document the + * watch follows was retired, which the next call reports however many frames it folds. */ - retire(this: Watch, context: Context): void { - if (!this.open || this.#retiring) return; - this.#retiring = true; - this.#ops.push(RETIREMENT_OPERATIONS); - this.#newest = { value: null, context }; + advance(change: WatchChange, context: Context): void { + if (!this.open) return; + this.#lone = this.#newestContext === undefined ? change : undefined; + this.#ops.push(change.ops); + this.#newestValue = change.value; + this.#newestContext = context; + if (change.retired) this.#retired = true; this.pending(); } - protected next(): WatchDelivery | undefined { - const newest = this.#newest; - if (newest === undefined) return undefined; - this.#newest = undefined; - const batch = this.#ops.take(); - // Values of a JSON watch are JSON; `T` is the source's narrower or wider view of them (e.g. a typed document). - const ops: readonly Op[] = batch.kind === "ops" ? batch.ops : [["r", newest.value as JsonValue]]; - // Nothing is queued after a retirement, so a retiring watch's pending call is its last. - return { value: newest.value, ops, context: newest.context, retires: this.#retiring }; + protected get pendingContext(): Context | undefined { + return this.#newestContext; } - protected invoke(listener: WatchListener, delivery: WatchDelivery, context: Context): Promise { - return listener(delivery.value, delivery.ops, context); + protected next(): WatchChange { + const lone = this.#lone; + // Set with the context, which says a frame is pending. + const value = this.#newestValue as T; + const batch = this.#ops.take(); + const retired = this.#retired; + this.#lone = undefined; + this.#newestValue = undefined; + this.#newestContext = undefined; + this.#retired = false; + // One frame never folds or overflows: its change is the call's. + if (lone !== undefined) return lone; + // Values of a JSON watch are JSON; `T` is the source's narrower or wider view of them (e.g. a typed document). + const ops: readonly Op[] = batch.kind === "ops" ? batch.ops : [["r", value as JsonValue]]; + return { value, ops, retired }; } protected release(): void { this.#ops.take(); - this.#newest = undefined; + this.#lone = undefined; + this.#newestValue = undefined; + this.#newestContext = undefined; + this.#retired = false; } } @@ -303,11 +332,15 @@ const MAX_PENDING_NATIVE_VALUES = 100; * The watch handle of a native source: every value in order, one call each; past 100 pending values they collapse to * one, `replace()` or by default the newest value. `detach` and `report` as for `Watch`. */ -export class NativeWatch - extends WatchLifecycle, Delivery> - implements NativeWatchHandle -{ - readonly #pending: Delivery[] = []; +export class NativeWatch extends WatchLifecycle> implements NativeWatchHandle { + /** + * Pending values in order, each followed by its frame's context, from `#head` on. Values are kept raw: the change a + * call gets is built when it is taken, so a value an overflow drops costs no object. A read index instead of + * `shift()` takes a backlog in constant time per value; taken slots are cleared at once, the array is emptied once + * all of it was taken, and its taken prefix dropped after 100 values. + */ + readonly #pending: Array = []; + #head = 0; #replace: (() => T) | undefined; constructor(value: T, detach: () => void, report: (error: unknown) => void, replace?: () => T) { @@ -318,26 +351,45 @@ export class NativeWatch /** Queue `value`, the source's new value. */ advance(value: T, context: Context): void { if (!this.open) return; - if (this.#pending.length < MAX_PENDING_NATIVE_VALUES) { - this.#pending.push({ value, context, retires: false }); + if (this.#pending.length - this.#head < 2 * MAX_PENDING_NATIVE_VALUES) { + this.#pending.push(value, context); } else { this.#pending.length = 0; - const replacement = this.#replace === undefined ? value : this.#replace(); - this.#pending.push({ value: replacement, context, retires: false }); + this.#head = 0; + this.#pending.push(this.#replace === undefined ? value : this.#replace(), context); } this.pending(); } - protected next(): Delivery | undefined { - return this.#pending.shift(); + protected get pendingContext(): Context | undefined { + // The slot after a value holds its context; past the end it is empty. + return this.#pending[this.#head + 1] as Context | undefined; } - protected invoke(listener: NativeWatchListener, delivery: Delivery, context: Context): Promise { - return listener(delivery.value, context); + protected next(): NativeWatchChange { + const head = this.#head; + const pending = this.#pending; + // The slot at `#head` holds a value. + const value = pending[head] as T; + if (head + 2 === pending.length) { + pending.length = 0; + this.#head = 0; + } else if (head + 2 >= 2 * MAX_PENDING_NATIVE_VALUES) { + // A backlog that never drains: drop the taken prefix, at most once per 100 values, so the array stays bounded. + pending.copyWithin(0, head + 2); + pending.length -= head + 2; + this.#head = 0; + } else { + pending[head] = undefined; + pending[head + 1] = undefined; + this.#head = head + 2; + } + return { value }; } protected release(): void { this.#pending.length = 0; + this.#head = 0; this.#replace = undefined; } } diff --git a/packages/chord/test/watch.test.ts b/packages/chord/test/watch.test.ts index 76b54401b..5ef530d60 100644 --- a/packages/chord/test/watch.test.ts +++ b/packages/chord/test/watch.test.ts @@ -1,7 +1,7 @@ // Tests for Chord's watch handles: the shared lifecycle through `Watch`, and what `NativeWatch` does differently. import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; -import { afterEach, describe, expect, expectTypeOf, it, vi } from "vitest"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { BACKGROUND_CONTEXT, type Context, @@ -12,7 +12,7 @@ import { import { applyImmutable, type Op, track } from "../src/delta/index.ts"; import { MAX_PENDING_OPS } from "../src/delta/pending.ts"; import type { JsonValue } from "../src/json.ts"; -import { NativeWatch, Watch, type WatchEnd } from "../src/watch.ts"; +import { NativeWatch, type NativeWatchChange, Watch, type WatchChange, type WatchEnd } from "../src/watch.ts"; const context = BACKGROUND_CONTEXT; @@ -25,6 +25,11 @@ function deferred(): { readonly promise: Promise; readonly resolve: () => return { promise, resolve: () => resolve() }; } +/** A source's frame that is not a retirement. */ +function frame(value: T, ops: readonly Op[]): WatchChange { + return { value, ops, retired: false }; +} + /** A watch whose detach and reports are recorded. */ function recordedWatch(value: T) { const recorded = { detached: 0, reported: [] as unknown[] }; @@ -39,9 +44,9 @@ function recordedWatch(value: T) { /** A started watch collecting its calls. */ function collectingWatch(value: T) { const { watch, recorded } = recordedWatch(value); - const frames: Array<{ value: T; ops: readonly Op[] }> = []; - watch.start(async (delivered, ops) => { - frames.push({ value: delivered, ops }); + const frames: Array<{ value: T; ops: readonly Op[]; retired?: true }> = []; + watch.start(async ({ value: delivered, ops, retired }) => { + frames.push(retired ? { value: delivered, ops, retired } : { value: delivered, ops }); }); return { watch, recorded, frames }; } @@ -69,21 +74,20 @@ const collect = (() => { describe("Watch delivery", () => { it("keeps the acquisition value until start, then delivers the frames since in one call", async () => { const { watch } = recordedWatch({ n: 0, text: "" }); - watch.advance({ n: 1, text: "" }, [["s", ["n"], 1]], context); + watch.advance(frame({ n: 1, text: "" }, [["s", ["n"], 1]]), context); watch.advance( - { n: 2, text: "a" }, - [ + frame({ n: 2, text: "a" }, [ ["s", ["n"], 2], ["a", ["text"], "a"], - ], + ]), context, ); - watch.advance({ n: 2, text: "ab" }, [["a", ["text"], "b"]], context); + watch.advance(frame({ n: 2, text: "ab" }, [["a", ["text"], "b"]]), context); await flush(); expect(watch.value).toEqual({ n: 0, text: "" }); const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { frames.push({ value, ops }); }); expect(frames).toEqual([]); @@ -103,14 +107,14 @@ describe("Watch delivery", () => { it("passes a frame's ops on by identity to a listener that keeps up", async () => { const { watch } = recordedWatch(0); const delivered: Array = []; - watch.start(async (_value, ops) => { + watch.start(async ({ ops }) => { delivered.push(ops); }); const first: readonly Op[] = [["r", 1]]; const second: readonly Op[] = [["r", 2]]; - watch.advance(1, first, context); + watch.advance(frame(1, first), context); await flush(); - watch.advance(2, second, context); + watch.advance(frame(2, second), context); await flush(); expect(delivered).toHaveLength(2); expect(delivered[0]).toBe(first); @@ -119,7 +123,7 @@ describe("Watch delivery", () => { it("never invokes the listener inline", async () => { const { watch, frames } = collectingWatch(0); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); expect(frames).toEqual([]); await flush(); expect(frames).toEqual([{ value: 1, ops: [["r", 1]] }]); @@ -130,16 +134,16 @@ describe("Watch delivery", () => { const gate = deferred(); const calls: string[] = []; const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { calls.push(`start ${JSON.stringify(value)}`); frames.push({ value, ops }); if (frames.length === 1) await gate.promise; calls.push(`end ${JSON.stringify(value)}`); }); - watch.advance({ text: "a" }, [["a", ["text"], "a"]], context); + watch.advance(frame({ text: "a" }, [["a", ["text"], "a"]]), context); await flush(); - watch.advance({ text: "ab" }, [["a", ["text"], "b"]], context); - watch.advance({ text: "abc" }, [["a", ["text"], "c"]], context); + watch.advance(frame({ text: "ab" }, [["a", ["text"], "b"]]), context); + watch.advance(frame({ text: "abc" }, [["a", ["text"], "c"]]), context); await flush(); expect(calls).toEqual(['start {"text":"a"}']); gate.resolve(); @@ -153,11 +157,11 @@ describe("Watch delivery", () => { const key = createContextKey("frame"); const controller = new AbortController(); const delivered: Context[] = []; - watch.start(async (_value, _ops, deliveryContext) => { + watch.start(async (_change, deliveryContext) => { delivered.push(deliveryContext); }); - watch.advance(1, [["r", 1]], withContextValue(key, "first", context)); - watch.advance(2, [["r", 2]], withContextValue(key, "second", withAbortSignal(controller.signal, context))); + watch.advance(frame(1, [["r", 1]]), withContextValue(key, "first", context)); + watch.advance(frame(2, [["r", 2]]), withContextValue(key, "second", withAbortSignal(controller.signal, context))); await flush(); expect(delivered).toHaveLength(1); expect(delivered[0]?.value(key)).toBe("second"); @@ -180,12 +184,12 @@ describe("Watch with a slow listener", () => { edit(change.state as Doc); const prepared = change.prepare(); tracker.adopt(prepared); - watch.advance(tracker.value, prepared.ops, context); + watch.advance(frame(tracker.value, prepared.ops), context); }; let replica: Doc = tracker.value; const calls: Array<{ commits: number; ops: number }> = []; let commits = 0; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { replica = applyImmutable(replica, ops); expect(replica).toEqual(value); calls.push({ commits, ops: ops.length }); @@ -217,7 +221,7 @@ function advanceDistinct(watch: Watch, count: number): JsonValue { const values: number[] = []; for (let index = 0; index < count; index++) { values.push(index); - watch.advance({ k: [...values] }, [["s", ["k", index], index]], context); + watch.advance(frame({ k: [...values] }, [["s", ["k", index], index]]), context); } return { k: values }; } @@ -240,11 +244,11 @@ describe("Watch overflow", () => { const { watch } = recordedWatch({ k: [] }); const gate = deferred(); const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { frames.push({ value, ops }); if (frames.length === 1) await gate.promise; }); - watch.advance({ k: [], first: true }, [["s", ["first"], true]], context); + watch.advance(frame({ k: [], first: true }, [["s", ["first"], true]]), context); await flush(); const newest = advanceDistinct(watch, MAX_PENDING_OPS + 1); gate.resolve(); @@ -259,8 +263,8 @@ describe("Watch overflow", () => { const { watch, frames } = collectingWatch({ k: [] }); advanceDistinct(watch, MAX_PENDING_OPS + 1); await flush(); - watch.advance({ n: 1 }, [["r", { n: 1 }]], context); - watch.advance({ n: 2 }, [["s", ["n"], 2]], context); + watch.advance(frame({ n: 1 }, [["r", { n: 1 }]]), context); + watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), context); await flush(); expect(frames.at(-1)).toEqual({ value: { n: 2 }, @@ -273,43 +277,62 @@ describe("Watch overflow", () => { }); describe("Watch retirement", () => { - it('delivers `[["r", null]]` as the last call and then ends retired', async () => { + it("reports a retirement in the next call with the value the source gives, and goes on", async () => { const { watch, frames, recorded } = collectingWatch(1); const settled = endOf(watch); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); await flush(); - watch.retire(context); - watch.advance(3, [["r", 3]], context); + watch.advance({ value: 2, ops: [], retired: true }, context); + await flush(); + watch.advance(frame(3, [["r", 3]]), context); await flush(); expect(frames).toEqual([ { value: 2, ops: [["r", 2]] }, - { value: null, ops: [["r", null]] }, + { value: 2, ops: [], retired: true }, + { value: 3, ops: [["r", 3]] }, ]); - expect(watch.value).toBeNull(); - expect(settled.end).toEqual({ reason: "retired" }); - expect(recorded.detached).toBe(1); + expect(watch.value).toBe(3); + expect(settled.end).toBeUndefined(); + expect(recorded.detached).toBe(0); + await watch.stop(); }); - it("drops pending frames, also an overflow, for the retirement", async () => { - for (const pending of [3, MAX_PENDING_OPS + 1]) { - const { watch } = recordedWatch({ k: [] }); - advanceDistinct(watch, pending); - watch.retire(context); - const frames: Array<{ value: JsonValue; ops: readonly Op[] }> = []; - watch.start(async (value, ops) => { - frames.push({ value, ops }); - }); - expect(await watch.closed).toEqual({ reason: "retired" }); - expect(frames).toEqual([{ value: null, ops: [["r", null]] }]); - } + it("folds a retirement and the frames around it into one call that reports it", async () => { + const { watch } = recordedWatch({ n: 0 }); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), context); + watch.advance({ value: { n: 1 }, ops: [], retired: true }, context); + watch.advance(frame({ m: 1 }, [["r", { m: 1 }]]), context); + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await flush(); + expect(changes).toEqual([{ value: { m: 1 }, ops: [["r", { m: 1 }]], retired: true }]); + watch.advance(frame({ m: 2 }, [["s", ["m"], 2]]), context); + await flush(); + expect(changes[1]).toEqual({ value: { m: 2 }, ops: [["s", ["m"], 2]], retired: false }); + await watch.stop(); }); - it("ends retired only on its source's retirement, never on a null value", async () => { + it("keeps reporting the retirement past an overflow", async () => { + const { watch } = recordedWatch({ k: [] }); + watch.advance({ value: { k: [] }, ops: [], retired: true }, context); + const newest = advanceDistinct(watch, MAX_PENDING_OPS + 1); + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await flush(); + expect(changes).toEqual([{ value: newest, ops: [["r", newest]], retired: true }]); + await watch.stop(); + }); + + it("treats a null value as a value, never as a retirement", async () => { const { watch, frames } = collectingWatch(1); const settled = endOf(watch); - watch.advance(null, [["r", null]], context); + watch.advance(frame(null, [["r", null]]), context); await flush(); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); await flush(); expect(frames).toEqual([ { value: null, ops: [["r", null]] }, @@ -319,24 +342,204 @@ describe("Watch retirement", () => { expect(await watch.stop()).toEqual({ reason: "stopped" }); }); - it("is only available to watches whose values include null", () => { - const check = (numbers: Watch, documents: Watch<{ n: number } | null>) => { - // @ts-expect-error `retire` delivers `null`, which a number watch cannot hold. - numbers.retire(context); - documents.retire(context); - }; - expectTypeOf(check).toBeFunction(); - }); - it("discards a retirement still pending when the source ends", async () => { const { watch } = recordedWatch(1); - watch.retire(context); + watch.advance({ value: 1, ops: [], retired: true }, context); watch.end(); expect(await watch.closed).toEqual({ reason: "ended" }); expect(watch.value).toBe(1); }); }); +describe("Watch end after delivery", () => { + it("takes no more frames, delivers what is pending, then ends `ended` and only then detaches", async () => { + const { watch, frames, recorded } = collectingWatch(1); + const settled = endOf(watch); + watch.advance({ value: 1, ops: [], retired: true }, context); + watch.endAfterDelivery(); + expect(recorded.detached).toBe(0); + // The source sends nothing more: a later frame is not taken. + watch.advance(frame(2, [["r", 2]]), context); + await flush(); + expect(frames).toEqual([{ value: 1, ops: [], retired: true }]); + expect(settled.end).toEqual({ reason: "ended" }); + expect(recorded.detached).toBe(1); + }); + + it("waits for start to deliver, and ends at once with nothing pending", async () => { + const { watch, recorded } = recordedWatch(1); + watch.advance(frame(2, [["r", 2]]), context); + watch.endAfterDelivery(); + await flush(); + const settled = endOf(watch); + await flush(); + expect(settled.end).toBeUndefined(); + expect(recorded.detached).toBe(0); + const values: JsonValue[] = []; + watch.start(async ({ value }) => { + values.push(value); + }); + expect(await watch.closed).toEqual({ reason: "ended" }); + expect(values).toEqual([2]); + expect(recorded.detached).toBe(1); + + const idle = collectingWatch(1); + idle.watch.endAfterDelivery(); + expect(await idle.watch.closed).toEqual({ reason: "ended" }); + expect(idle.frames).toEqual([]); + expect(idle.recorded.detached).toBe(1); + }); + + it("ends a never-started watch at its source's end, dropping what is pending", async () => { + const { watch, recorded } = recordedWatch(1); + watch.advance({ value: 1, ops: [], retired: true }, context); + watch.endAfterDelivery(); + const failure = new Error("source failed"); + watch.end({ error: failure }); + expect(await watch.closed).toEqual({ reason: "failed", error: failure }); + expect(recorded.detached).toBe(1); + expect(() => watch.start(async () => {})).toThrow("Watch is stopped"); + }); + + /** A started watch whose listener is in a call held by `gate`, with nothing pending, then `endAfterDelivery()`. */ + async function endingWhileRunning(settle: () => Promise) { + const { watch, recorded } = recordedWatch(0); + const gate = deferred(); + const values: JsonValue[] = []; + let returned = false; + watch.start(async ({ value }) => { + values.push(value); + await gate.promise; + await settle(); + returned = true; + }); + watch.advance({ value: 1, ops: [["r", 1]], retired: true }, context); + await flush(); + expect(values).toEqual([1]); + watch.endAfterDelivery(); + const settled = endOf(watch); + await flush(); + return { watch, recorded, values, settled, release: gate.resolve, returned: () => returned }; + } + + it("ends `ended` only after the running call returns when nothing is pending, detached then", async () => { + const ending = await endingWhileRunning(async () => {}); + expect(ending.settled.end).toBeUndefined(); + expect(ending.recorded.detached).toBe(0); + ending.release(); + expect(await ending.watch.closed).toEqual({ reason: "ended" }); + expect(ending.returned()).toBe(true); + expect(ending.values).toEqual([1]); + expect(ending.recorded.detached).toBe(1); + }); + + it("ends `listener_error` and reports it when the running call throws after `endAfterDelivery()`", async () => { + const failure = new Error("listener failed"); + const ending = await endingWhileRunning(async () => { + throw failure; + }); + expect(ending.settled.end).toBeUndefined(); + ending.release(); + expect(await ending.watch.closed).toEqual({ reason: "listener_error", error: failure }); + expect(ending.recorded.reported).toEqual([failure]); + expect(ending.recorded.detached).toBe(1); + }); + + it("ends `stopped` when stopped while ending, without awaiting the running call", async () => { + const ending = await endingWhileRunning(async () => {}); + expect(await ending.watch.stop()).toEqual({ reason: "stopped" }); + expect(ending.recorded.detached).toBe(1); + expect(ending.returned()).toBe(false); + ending.release(); + await flush(); + expect(await ending.watch.closed).toEqual({ reason: "stopped" }); + }); +}); + +describe("Watch change objects", () => { + /** A frame context carrying `payload` and an abort signal, as a producer's is. */ + function producerContext(payload: object) { + const controller = new AbortController(); + return withAbortSignal( + controller.signal, + withContextValue(createContextKey("payload"), payload, context), + ); + } + + it("hands a listener plain data only: `{ value, ops, retired }`, cloneable and serializable", async () => { + const { watch } = recordedWatch({ n: 0 }); + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + watch.advance(frame({ n: 1 }, [["s", ["n"], 1]]), producerContext({})); + await flush(); + // Folded: the watch builds the change. + watch.advance(frame({ n: 2 }, [["s", ["n"], 2]]), producerContext({})); + watch.advance({ value: { n: 2 }, ops: [], retired: true }, producerContext({})); + await flush(); + expect(changes).toHaveLength(2); + for (const change of changes) { + expect(Object.keys(change)).toEqual(["value", "ops", "retired"]); + expect(structuredClone(change)).toEqual(change); + expect(JSON.parse(JSON.stringify(change))).toEqual(change); + } + expect(changes[1]).toEqual({ value: { n: 2 }, ops: [["s", ["n"], 2]], retired: true }); + await watch.stop(); + }); + + it("passes a frame delivered alone on as the source made it", async () => { + const { watch } = recordedWatch(0); + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + const alone = frame(1, [["r", 1]]); + watch.advance(alone, context); + await flush(); + expect(changes[0]).toBe(alone); + await watch.stop(); + }); + + it("keeps no frame context alive through a change a listener keeps", async () => { + const kept: Array | NativeWatchChange> = []; + /** Deliver one frame whose context alone references `payload`, keep the change, and stop the watch. */ + const deliver = async (native: boolean): Promise> => { + const payload = { big: "x".repeat(1000) }; + if (native) { + const watch = new NativeWatch( + 0, + () => {}, + () => {}, + ); + watch.start(async (change) => { + kept.push(change); + }); + watch.advance(1, producerContext(payload)); + await flush(); + expect(Object.keys(kept.at(-1)!)).toEqual(["value"]); + expect(structuredClone(kept.at(-1))).toEqual({ value: 1 }); + await watch.stop(); + } else { + const { watch } = recordedWatch(0); + watch.start(async (change) => { + kept.push(change); + }); + watch.advance(frame(1, [["r", 1]]), producerContext(payload)); + await flush(); + await watch.stop(); + } + return new WeakRef(payload); + }; + for (const native of [false, true]) { + const payload = await deliver(native); + await collect(); + expect(kept).toHaveLength(native ? 2 : 1); + expect(payload.deref()).toBeUndefined(); + } + }); +}); + describe("Watch ends", () => { it("stops idempotently and rejects a repeated or late start", async () => { const { watch, recorded } = recordedWatch(0); @@ -353,7 +556,7 @@ describe("Watch ends", () => { it("ends `ended` or `failed` from its source, dropping pending frames", async () => { const { watch, frames } = collectingWatch(0); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); watch.end(); await flush(); expect(frames).toEqual([]); @@ -370,14 +573,14 @@ describe("Watch ends", () => { const { watch } = recordedWatch(0); const gate = deferred(); const calls: string[] = []; - watch.start(async (value) => { + watch.start(async ({ value }) => { calls.push(`start ${value}`); await gate.promise; calls.push(`end ${value}`); }); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); await flush(); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); expect(await watch.stop()).toEqual({ reason: "stopped" }); gate.resolve(); await flush(); @@ -407,14 +610,14 @@ describe("Watch ends", () => { const failure = new Error("listener failed"); const gate = deferred(); const values: JsonValue[] = []; - watch.start(async (value) => { + watch.start(async ({ value }) => { values.push(value); await gate.promise; throw failure; }); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); await flush(); - watch.advance(2, [["r", 2]], context); + watch.advance(frame(2, [["r", 2]]), context); gate.resolve(); expect(await watch.closed).toEqual({ reason: "listener_error", error: failure }); expect(values).toEqual([1]); @@ -425,7 +628,7 @@ describe("Watch ends", () => { it("wraps a thrown non-Error in an Error", async () => { const { watch } = recordedWatch(0); watch.start(() => Promise.reject("plain")); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); const end = await watch.closed; expect(end.reason).toBe("listener_error"); expect(end.reason === "listener_error" && end.error.message).toBe("plain"); @@ -446,7 +649,7 @@ describe("Watch references", () => { const listener = async (): Promise => {}; watch.observeCancellation(controller.signal); watch.start(listener); - watch.advance(1, [["r", 1]], context); + watch.advance(frame(1, [["r", 1]]), context); await flush(); end(watch); return { @@ -469,7 +672,7 @@ describe("Watch references", () => { (watch: Watch) => void watch.stop(), (watch: Watch) => watch.cancel(), (watch: Watch) => watch.end(), - (watch: Watch) => watch.retire(context), + (watch: Watch) => watch.endAfterDelivery(), ]) { const ended = await acquire(end); await flush(); @@ -480,6 +683,59 @@ describe("Watch references", () => { } }); + it("keeps no ops a fold or an overflow dropped, while it waits", async () => { + /** Feed an unstarted watch four tracker batches of 4,000 sets each: the third overflows, the fourth is ignored. */ + const overflow = () => { + const initial: Record = {}; + for (let i = 0; i < 16_000; i++) initial[`k${i}`] = 0; + const tracker = track>(initial); + const watch = new Watch( + initial, + () => {}, + () => {}, + ); + const batches: Array> = []; + for (let round = 0; round < 4; round++) { + const change = tracker.beginChange(); + for (let i = round * 4_000; i < (round + 1) * 4_000; i++) change.state[`k${i}`] = 1; + const prepared = change.prepare(); + tracker.adopt(prepared); + batches.push(new WeakRef(prepared.ops)); + watch.advance(frame(tracker.value, prepared.ops), context); + } + return { watch, batches, newest: tracker.value }; + }; + const { watch, batches, newest } = overflow(); + await collect(); + expect(batches.map((batch) => batch.deref() === undefined)).toEqual([true, true, true, true]); + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await flush(); + expect(changes).toEqual([{ value: newest, ops: [["r", newest]], retired: false }]); + await watch.stop(); + }); + + it("delivers a native backlog that never drains in order", async () => { + const watch = new NativeWatch( + 0, + () => {}, + () => {}, + ); + const values: number[] = []; + let next = 50; + // Each call queues one more value behind a backlog of 50, so the queue never empties for 1,000 values. + watch.start(async ({ value }) => { + values.push(value); + if (next < 1_000) watch.advance(next++, context); + }); + for (let n = 0; n < 50; n++) watch.advance(n, context); + await flush(); + expect(values).toEqual(Array.from({ length: 1_000 }, (_, n) => n)); + await watch.stop(); + }); + it("releases its pending queue and leaf index once delivered or ended", async () => { /** Two pending frames, folded, whose ops alone reference `payload`; `settle` delivers or ends them. */ const pending = async (settle: (watch: Watch, gate: () => void) => Promise) => { @@ -494,10 +750,10 @@ describe("Watch references", () => { watch.start(async () => { if (++calls === 1) await gate.promise; }); - watch.advance(1, [["s", ["n"], 1]], context); + watch.advance(frame(1, [["s", ["n"], 1]]), context); await flush(); - watch.advance(2, [["s", ["o"], payload]], context); - watch.advance(3, [["s", ["n"], 3]], context); + watch.advance(frame(2, [["s", ["o"], payload]]), context); + watch.advance(frame(3, [["s", ["n"], 3]]), context); await settle(watch, gate.resolve); return { watch, payload: new WeakRef(payload) }; }; @@ -529,7 +785,7 @@ describe("NativeWatch", () => { ); const key = createContextKey("frame"); const delivered: Array<{ value: { run: () => number }; frame: string | undefined }> = []; - watch.start(async (value, deliveryContext) => { + watch.start(async ({ value }, deliveryContext) => { delivered.push({ value, frame: deliveryContext.value(key) }); }); watch.advance(first, withContextValue(key, "first", context)); @@ -551,7 +807,7 @@ describe("NativeWatch", () => { ); for (let n = 1; n <= 101; n++) watch.advance(n, context); const values: number[] = []; - watch.start(async (value) => { + watch.start(async ({ value }) => { values.push(value); }); await flush(); @@ -567,7 +823,7 @@ describe("NativeWatch", () => { ); for (let n = 1; n <= 101; n++) watch.advance(`value ${n}`, context); const values: string[] = []; - watch.start(async (value) => { + watch.start(async ({ value }) => { values.push(value); }); await flush(); @@ -592,6 +848,18 @@ describe("NativeWatch", () => { ended.end(); expect(await ended.closed).toEqual({ reason: "ended" }); + const finishing = create(); + const finished: number[] = []; + finishing.advance(1, context); + finishing.advance(2, context); + finishing.endAfterDelivery(); + finishing.advance(3, context); + finishing.start(async ({ value }) => { + finished.push(value); + }); + expect(await finishing.closed).toEqual({ reason: "ended" }); + expect(finished).toEqual([1, 2]); + const error = new Error("source failed"); const failed = create(); failed.end({ error }); diff --git a/packages/durable/CHANGELOG.md b/packages/durable/CHANGELOG.md index a6431c209..ed67bba7b 100644 --- a/packages/durable/CHANGELOG.md +++ b/packages/durable/CHANGELOG.md @@ -4,21 +4,30 @@ ### Breaking Changes -- `Storage.commit(writes, adopt, context)` takes an `adopt(seq)` callback, the Session's in-memory adoption of the batch: a storage calls it exactly once, synchronously, once the batch is durable and before its next commit starts, then resolves with `seq`, and never for a commit that fails before the batch is durable. Memory, SQLite, JSONL, and IndexedDB storages do; custom storages and wrappers must call or forward it, and the storage conformance suite checks it. A commit that rejects after its adoption fails the Session without rolling the adopted changes back. +- `Storage.commit(writes, adopt, context)` takes an `adopt(seq)` callback, the Session's in-memory adoption and publication of the batch: a storage calls it exactly once, synchronously, once the batch is durable and before its next commit starts, then resolves with `seq`, and never for a commit that fails before the batch is durable. Memory, SQLite, JSONL, and IndexedDB storages do; custom storages and wrappers must call or forward it, and the storage conformance suite checks it. A commit that rejects after its adoption is published and fails the Session without rolling the adopted changes back. - `Storage.mintId()`'s contract is narrowed to IDs greater than every ID minted since the storage was opened and every ID stored, also across reopen. The bundled storages already behave so. -- Watches use Chord's `WatchHandle` and `WatchEnd` (Durable re-exports both). End reasons are `stopped`, `cancelled`, `retired`, `ended` (was `session_closed`), `listener_error`, and `failed` (was `session_failed`). Retirement is an explicit event of the watched document incarnation (last frame `[["r", null]]`, then `retired`), never inferred from a `null` value. An ended watch drops its listener and every reference into its Session, so a stopped watch that is still referenced keeps only its last value. +- Watches use Chord's `WatchHandle` and `WatchEnd` (Durable re-exports both, with `WatchChange` and `NativeWatchChange`). End reasons are `stopped`, `cancelled`, `ended` (was `session_closed`), `listener_error`, and `failed` (was `session_failed`); `retired` is gone (see below). An ended watch drops its listener and every reference into its Session, so a stopped watch that is still referenced keeps only its last value. +- Watch listeners take one change object before the context: `watch.start(async ({ value, ops, retired }, context) => …)` for document, conversation view (`Conversation.watch()`), and task graph watches, and `stream.start(async ({ value: events }, context) => …)` for `watchEvents()`. Migration: `start(async (value, ops, ctx) => …)` becomes `start(async ({ value, ops }, ctx) => …)`, and an event listener `(events, ctx)` becomes `({ value: events }, ctx)`. +- Document watches follow their address (kind, scope, and key) instead of one incarnation: `DocumentWatch` is `WatchHandle>` (no `null`). A retirement no longer ends the watch with a last frame `[["r", null]]` and `retired`: the next call has `retired: true`, the last value the watch was given, and no ops (folded with what is pending; a change made in the retiring commit itself is not shown), and a document created at the address later arrives on the same watch as `[["r", value]]`, possibly in the same call. A task document's watch ends `ended` once the retirement call of the commit that makes its task terminal has returned; Session close or failure still ends it first if that call was not delivered yet (a watch never started). Acquiring an absent document still gives `undefined`. A watch pins the incarnation at its address, nothing while it is absent. Migration: test `change.retired` instead of a `null` value or a `retired` end, and stop a watch whose document may be retired for good, as it otherwise stays until the Session ends. - Document, conversation view, and task graph watches deliver the commits made while their listener runs (or before `start()`) in one callback: the newest value, the commits' operations folded (appends to one string join, sets of one field keep the last) and the newest commit's Context. Past 10,000 pending operations the callback gets one root replacement `[["r", newest]]`. Before, each commit was its own callback and more than 100 pending commits became a root replacement. `watchEvents()` still delivers every event batch in order, with one `snapshot` past 100 pending batches. - Removed the Chord state getters `Session.documentState()`, `Conversation.viewState()`, and `Harness.taskGraph()`, and the `DocumentState` type; use `watchDoc()`, `Conversation.watch()`, and `Harness.watchTaskGraph()`. Durable no longer uses Chord's `replicatedState()`. -- Partial answers and running tool output are stored later than they are shown: their throttled commits are deferred (`CommitPersistence`, spec §4.1), so watches, views, and agent events see each at once, while Storage gets them together with the next commit that reads `pi.live` (the end of the answer, a tool settling, a run ending) and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). With 32 streams over SQLite this cuts storage transactions about 5.7x and disk writes about 2.9x for turns with parallel tools, 27x and 9x for long answers, and CPU by about a quarter. A crash now loses up to about a second of streamed content instead of 100 ms; a stream that crashes before its first store leaves no aborted entry, and that attempt's usage is not recorded. `subscribeCommits()` gets these commits only once they are stored, with exactly what was stored (held operations first), and gets a publication for a deadline's store too; it no longer sees every partial. Migration: a `subscribeCommits()` listener that showed live progress should use `watchDoc()`, `Conversation.watch()`, or `watchEvents()`; one that mirrors Storage keeps working; set `settings.progress.persistWithinMs: 0` to store progress right after each commit, about as before (one storage transaction per partial). +- Partial answers and running tool output are stored later than they are shown: their throttled commits are deferred (`CommitPersistence`, spec §4.1), so watches, views, and agent events see each at once, while Storage gets them together with the next commit that reads `pi.live` (the end of the answer, a tool settling, a run ending) and at the latest after `settings.progress.persistWithinMs` (default 1,000 ms). With 32 streams over SQLite this cuts storage transactions about 5.7x and disk writes about 2.9x for turns with parallel tools, 27x and 9x for long answers, and CPU by about a quarter. A crash now loses up to about a second of streamed content instead of 100 ms; a stream that crashes before its first store leaves no aborted entry, and that attempt's usage is not recorded. `subscribeCommits()` gets these commits only once they are stored, with exactly what was stored (held operations first; a document the commit retires shows as its retirement only, without the operations stored before it), and gets a publication for a deadline's store too; it no longer sees every partial. Migration: a `subscribeCommits()` listener that showed live progress should use `watchDoc()`, `Conversation.watch()`, or `watchEvents()`; one that mirrors Storage keeps working; set `settings.progress.persistWithinMs: 0` to store progress right after each commit, about as before (one storage transaction per partial). +- The change type of `FileSystem.watch()` in `@earendil-works/pi-durable/env` is renamed from `WatchChange` to `FileChange` (same shape: `{ paths }`, `{ overflow: true }`, or `{ error }`), so `WatchChange` names only a watch listener's change. Migration: import `type FileChange` from `@earendil-works/pi-durable/env` where `WatchChange` was imported from it. +- `HostCallback` gains `"telemetry"`: a `host_error` report about a telemetry adapter that threw while a commit was published. Exhaustive switches over `HostCallback` need a case for it. ### Added - Deferred commits: `Session.commit()`, `Conversation.commit()`, `TaskRuntime.commit()`, and a tool's `api.commit()` take `{ persistence: { kind: "deferred", withinMs? } }`. A commit that changes only existing conversation documents with `history: "latest"` and `fork: "initial"` is then adopted in memory and shown to watches at once, and stored with the next commit that reads or writes the document, before the document leaves memory, at close, or within `withinMs` (default `settings.progress.persistWithinMs`, or `createSession(storage, { persistWithinMs })`); any other write makes it immediate. A crash loses what is held. See spec §4.1. +- Synchronous observers for adapters, such as a Chord source over a Durable view: `Session.observeDoc(token, ...scope, observer, context)`, `Harness.observeConversation(id, observer, context)`, and `Harness.observeTaskGraph(observer, context)` attach an `Observer` (`advance(change, context)`, `end(failure)`) atomically on the Session line, like a watch, and return an `Observation` (`value`, `detach()`; `observeDoc` gives `undefined` for an absent document). `advance` gets every change unfolded as memory adopts it, for a stored commit inside Storage's `adopt` and for a deferred one before it is stored, so it must only record it: no awaits, no Session calls, no throws. A document observer gets one call per change at its address (two for a commit that retires and recreates it), the task graph also a call outside any commit when a registry change or a failed migration changes a `blocked` reason, and a change can arrive before the attaching promise resolves. The attaching context governs only the attachment; a closed Session or an invalid argument rejects the promise. One that throws is reported as `listener_error`, detached, and ended with the error. `end` comes once, at Session close or failure, or after a task document's retirement with its task. Document, view, and task graph watches are built on them (spec §9.1). - `HarnessOptions.readOnly` opens a Harness as a viewer of a session another process drives: open leaves running tasks as committed instead of recovering them, every commit with writes rejects with `ReadOnlySession` before it reaches Storage, and `resume()` and every call that would run tasks reject with it. ### Changed - Document, conversation view, task graph, and event watches use Chord's watch handles (`Watch`, and `NativeWatch` for event batches). +- A stored commit is published when Storage adopts it, inside `Storage.commit()`: the commit's telemetry events and Durable's own observers (document watches, conversation views, the task graph, the scheduler, and waits) run inside `adopt`, so two views never show one commit apart. `subscribeCommits()` listeners run one microtask later, outside the storage's synchronous step, before every wait the commit resolves and every abort signal it fires; they have no order relative to watch listeners and `onWake`. With a storage that works after adopting (JSONL's sidecar reclaim), all of these run before `Storage.commit()` resolves. +- `subscribeCommits()` listeners are read when a commit's dispatch runs: a listener added between adoption and the dispatch gets the commit only when one was registered at adoption; one added after the dispatch ran misses it. +- The abort signal of a newly marked running task fires after `subscribeCommits()` listeners instead of before them. +- A throwing Durable commit observer fails the Session after `Storage.commit()` returned, after the other observers and the host listeners got the commit; the commit still stands. When Storage also rejects that commit after adopting it, the storage error fails the Session and the observer's error is reported as `listener_error`. ### Fixed @@ -29,6 +38,8 @@ - Fixed partial commits of a streamed answer copying every growing text, thinking and tool-call argument string in full on each commit (about 20 s of CPU and a 100 MB heap peak for a 500 KB tool call): they now append new text with Chord's `appendText()`, and apply the argument changes of their own parse of the streamed argument text ([#9265](https://github.com/earendil-works/pi/issues/9265)) - Fixed stored assistant messages lacking `thinkingLevel`: a generation records the Pi thinking level its request asked for on the response, also after polling a deferred one, as pi-agent-core's agent loop does - Fixed partial commits polluting `Object.prototype` when a streamed tool call's arguments gained a `__proto__` member after an earlier commit: the merge read the missing member through to `Object.prototype` and wrote into it +- Fixed a commit that Storage rejects after adopting it (a failed JSONL reclaim) publishing nothing: it is published and its waits resolve before the Session fails, so a replica built from `subscribeCommits()` stays equal to Storage and `pi.durable.tasks.active` stays balanced. +- Fixed a throwing telemetry adapter (a commit's `pi.durable.entry` or `pi.durable.submission` event, `pi.durable.tasks.active`) rejecting a durable commit and leaving the views and the scheduler without it: it is reported as `host_error` with callback `"telemetry"`, and the commit resolves. A throwing report metric or report event is dropped. ## [1.4.0] - 2026-10-21 diff --git a/packages/durable/README.md b/packages/durable/README.md index 0ce9885bb..d10263776 100644 --- a/packages/durable/README.md +++ b/packages/durable/README.md @@ -478,7 +478,7 @@ Everything a UI needs is committed state. `watch()` returns a watch of the conve ```typescript const watch = await root.watch(context); render(watch.value); // the state at attachment -watch.start(async (value, ops) => { +watch.start(async ({ value, ops }) => { // value.entries: the active transcript // value.docs["pi.live"]: the running generation (streamed partial, retry, deferred) and tool calls (output, details) // value.docs["pi.inbox"], value.docs["pi.usage"], value.docs["pi.agent"], value.docs["pi.provider"] @@ -488,7 +488,9 @@ watch.start(async (value, ops) => { // later: await watch.stop(); ``` -Watches are Chord's `WatchHandle` (exported by `@earendil-works/chord`; Durable re-exports it). A watch ends once, and `closed` (or `stop()`) gives the reason: `stopped`, `cancelled` (its acquisition context was cancelled), `retired` (a document watch whose incarnation retired, after it delivered `[["r", null]]`), `ended` (the Harness closed), `listener_error`, or `failed` (the Harness failed). An ended watch keeps its last value and no longer references the Harness. +Watches are Chord's `WatchHandle` (exported by `@earendil-works/chord`; Durable re-exports it). The listener gets one change per call, `{ value, ops, retired }`, and the newest commit's context. A watch ends once, and `closed` (or `stop()`) gives the reason: `stopped`, `cancelled` (its acquisition context was cancelled), `ended` (the Harness closed, or a task document's task ended), `listener_error`, or `failed` (the Harness failed). An ended watch keeps its last value and no longer references the Harness. + +Adapters that must change in the same step as Durable's memory, such as a Chord source over a view, attach a synchronous observer instead of a watch: `harness.observeConversation(id, observer, context)`, `harness.observeTaskGraph(observer, context)`, and `harness.observeDoc(token, ...scope, observer, context)` return the current `value` and `detach()`, and call `observer.advance(change, context)` with every later change, unfolded, as memory adopts it: inside the storage's commit for a stored commit, at once for a deferred one (before it is stored). A document observer gets one call per change at its address, so a commit that retires and recreates the document gives two; the task graph also advances without a commit when a registry change alters a task's `blocked` reason. A change can arrive before the `observeXXX()` promise resolves; its ops apply from the observation's `value`. `observer.end(failure)` comes once, when the Harness closes or fails, or, for a task document, right after the retirement of the commit that ends its task. `advance` only records the change: no awaits, no Harness calls, no throws, and quick, since every commit waits for it. One that throws is reported, detached, and ended with the error. Watches are built on these observers. Commits that land while a callback runs are delivered together in the next callback: the newest view and the operations of all of them, folded (appends to one string join, sets of one field keep the last). Past 10,000 pending operations the callback gets one root replacement `[["r", view]]` holding the whole newest view instead. A client that joins late or reconnects starts from the current view; nothing is replayed. @@ -591,7 +593,7 @@ import { watchEvents } from "@earendil-works/pi-durable"; const stream = await watchEvents(harness, root.id, context); initialize(stream.snapshot); // entries, run, in-flight generation, tools, compactions, inbox, agent, usage -stream.start(async (events) => { +stream.start(async ({ value: events }) => { for (const event of events) console.log(JSON.stringify(event)); }); ``` @@ -708,7 +710,7 @@ decide: async (task, runtime, context) => { ```typescript const graph = await harness.watchTaskGraph(context); -graph.start(async (value) => { +graph.start(async ({ value }) => { for (const node of Object.values(value.tasks)) { const status = node.state.status === "waiting" ? `waiting on ${node.state.on.join(", ")}` : node.state.status; console.log(`${node.id} ${node.kind} ${status}`, node.owner ?? `conversation ${node.conversationId}`); @@ -753,9 +755,9 @@ await root.commit( ); ``` -Watches and the view see it at once. Storage gets it with the next commit that reads or writes the document, before the document leaves memory, at close, and at the latest about `withinMs` later; `subscribeCommits()` sees it then, with exactly what was stored. A crash loses what was not stored yet, so derive nothing durable from such a value outside a commit that reads it. Only a commit that changes nothing but existing documents with `history: "latest"` and `fork: "initial"` is deferred; anything else in it, such as an entry, a new document, or a task state, stores it at once. +Watches and the view see it at once. Storage gets it with the next commit that reads or writes the document, before the document leaves memory, at close, and at the latest about `withinMs` later; `subscribeCommits()` sees it then, with exactly what was stored (a document the commit retires shows as its retirement only). A crash loses what was not stored yet, so derive nothing durable from such a value outside a commit that reads it. Only a commit that changes nothing but existing documents with `history: "latest"` and `fork: "initial"` is deferred; anything else in it, such as an entry, a new document, or a task state, stores it at once. -`harness.watchDoc()` watches one document like the view above. `HarnessOptions.conversationCreated(tx, conversation)` runs in every commit that creates or forks a conversation, including a tool's raw `tx.createConversation()`, so every conversation gets your documents; `init` in `createConversation()`, `fork()`, and `root()` writes per-call data in the same commit. An extension's tools, sections, and hooks read their own documents through `api` or `input.read`, and treat an absent one as its default ([`10-extension-state.ts`](test/examples/10-extension-state.ts)). +`harness.watchDoc()` watches one document like the view above. A document watch follows its address: a retirement is a call with `retired: true`, the last value the watch was given (a change made in the retiring commit itself is not shown), and no ops, and a document created there later arrives as `[["r", value]]` on the same watch. A task document's watch ends `ended` with its task: once the retirement call of the commit that ends the task has returned, or at the Harness's close if that call was never delivered (a watch never started); an observer of it gets `end(undefined)` right after the retirement. `HarnessOptions.conversationCreated(tx, conversation)` runs in every commit that creates or forks a conversation, including a tool's raw `tx.createConversation()`, so every conversation gets your documents; `init` in `createConversation()`, `fork()`, and `root()` writes per-call data in the same commit. An extension's tools, sections, and hooks read their own documents through `api` or `input.read`, and treat an absent one as its default ([`10-extension-state.ts`](test/examples/10-extension-state.ts)). ## Usage and Cost @@ -807,7 +809,7 @@ await database.transaction(async (transaction) => { }); ``` -A custom backend's `commit(writes, adopt, context)` calls `adopt(seq)` once the batch is durable, synchronously and before its next commit starts, and then resolves with `seq`; a commit that fails before the batch is durable never calls it. `adopt` is the Session's in-memory adoption of the batch, so memory never shows a partial log. Custom backends can run the shared conformance suite, which checks this too, with any Vitest- or Jest-compatible runner: +A custom backend's `commit(writes, adopt, context)` calls `adopt(seq)` once the batch is durable, synchronously and before its next commit starts, and then resolves with `seq`; a commit that fails before the batch is durable never calls it. `adopt` is the Session's in-memory adoption and publication of the batch, so memory and every view never show a partial log: inside it the Session's own observers run (views, task graph, scheduler, waits), and `subscribeCommits()` listeners, watch listeners and the code the commit wakes follow in microtasks, so at the backend's next `await` if it still works after `adopt` (as JSONL reclaims sidecars), before its `commit` resolves. `adopt` calls no Storage method and never throws; once it ran, the commit is published even if the backend then rejects, and the Session then fails. Custom backends can run the shared conformance suite, which checks this too, with any Vitest- or Jest-compatible runner: ```typescript import { registerStorageConformance } from "@earendil-works/pi-durable/testing"; diff --git a/packages/durable/docs/spec.md b/packages/durable/docs/spec.md index 3af8ff115..f82c2ac37 100644 --- a/packages/durable/docs/spec.md +++ b/packages/durable/docs/spec.md @@ -64,10 +64,11 @@ The core rule is: Required invariants: 1. One Session commit is atomic across all record and document writes. -2. A document update is published only after its storage commit succeeds, - except a deferred commit's (section 4.1): it is published to watches and the - conversation view when memory adopts it, and to `subscribeCommits()` once it - is stored. +2. A stored commit is published once Storage made it durable, when Storage + adopts it (section 4), also when `Storage.commit()` then rejects; a commit + that fails before it is durable publishes nothing. A deferred commit + (section 4.1) is published to watches and the conversation view when memory + adopts it, and to `subscribeCommits()` once it is stored. 3. All visible progress is durable, except what a deferred commit holds: what watches see can be ahead of Storage by up to that commit's window (`withinMs`), and a crash loses it. A commit is deferred only when its caller @@ -81,9 +82,13 @@ Required invariants: Values assigned into a draft are copied by value and must be strict JSON. 7. The mutation line remains held through storage settlement and committed-state adoption. Commit/close observers run synchronously only to capture immutable - state; document-state and watch user callbacks run later, off the line. -8. A storage error is fatal to the open Session (section 10.1). It publishes - nothing and must be reopened. Preparation and checkpoint failures occur before + state; document-state and watch user callbacks run later, as microtasks the + line never waits for, and never inside Storage's synchronous `adopt`. With a + storage that works after `adopt` (JSONL), they may run while the line is + still held by the commit that published. +8. A storage error is fatal to the open Session (section 10.1), which must be + reopened. A commit it fails publishes nothing, unless Storage had already + made it durable and adopted it. Preparation and checkpoint failures occur before storage admission and roll back normally. ## 2. Core records @@ -397,7 +402,8 @@ type HarnessOptions = { ) => (() => void) | undefined; /** * Replaces the timer for sleeping tasks (section 5.6): called with `Harness.wake()` after open and whenever it - * changes, off the Session line; the host arms its own wakeup and calls `resume()` when it fires. + * changes, in a microtask after the commit that changed it was adopted; the host arms its own wakeup and calls + * `resume()` when it fires. */ readonly onWake?: (wake: Wake) => void; /** @@ -607,7 +613,7 @@ type Wake = { readonly active: boolean; readonly at?: number }; /** What a report is about (section 10.1). */ type ReportTask = { readonly id: TaskId; readonly kind: string; readonly conversationId: ConversationId }; -type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake"; +type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake" | "telemetry"; type ReportInfo = | { readonly type: "session_failed" } | { @@ -712,6 +718,9 @@ interface Harness extends Session { usage(context: Context): Promise; /** Live tasks as a structural view (section 9.5), as a watch. */ watchTaskGraph(context: Context): Promise; + /** Synchronous observers of a conversation view and of the task graph, for adapters (section 9.1). */ + observeConversation(id: ConversationId, observer: Observer, context: Context): Promise>; + observeTaskGraph(observer: Observer, context: Context): Promise>; } declare const Harness: { @@ -1320,6 +1329,14 @@ interface Session extends DocumentObserver { snapshotAsOf(token: RewindableConversationDocToken, conversationId: ConversationId, at: EntryId, context: Context): Promise | undefined>; snapshotAsOf(token: RewindableConversationDocFamilyToken, conversationId: ConversationId, key: string, at: EntryId, context: Context): Promise | undefined>; + + /** A synchronous observer of one document address, for adapters (section 9.1); the scope arguments as for `snapshot()`. */ + observeDoc(token: SessionDocToken, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: ConversationDocToken, conversationId: ConversationId, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: TaskDocToken, taskId: TaskId, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: SessionDocFamilyToken, key: string, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: ConversationDocFamilyToken, conversationId: ConversationId, key: string, observer: Observer>, context: Context): Promise> | undefined>; + observeDoc(token: TaskDocFamilyToken, taskId: TaskId, key: string, observer: Observer>, context: Context): Promise> | undefined>; } interface Tx { @@ -1505,11 +1522,17 @@ commit deferred and holdable (section 4.1) release the line; invoke listeners later otherwise, right before Storage admission take the held ops of every document the commit acquired and put them in front of its own -storage succeeds - adopt every prepared change by pointer swap and enqueue immutable revision/ops publication - commit listeners, including conversation view mounts (section 9.3), capture it synchronously - release the line; invoke listeners later -storage fails +storage makes it durable and calls adopt(seq), inside Storage.commit() (section 4) + adopt every prepared change by pointer swap; record its telemetry events + queue one microtask that calls subscribeCommits() listeners with what was stored + the Session's own observers, including watches and view mounts (section 9.3), capture it + synchronously; the waits they resolve and the aborts they queue run after that microtask +storage resolves + fail the Session if an own observer threw (the commit stands); release the line +storage rejects after adopt(seq) + the commit stays adopted and published; the storage error fails the Session (section 10.1), + and an own observer's throw is reported +storage fails before adopt(seq) abort every prepared change, fail the Session (section 10.1), and publish nothing ``` @@ -1721,24 +1744,67 @@ sets them for a plain Session; the Harness reads its settings at each use. A Session commit callback may be asynchronous. It owns the Session mutation line through callback execution, preparation, storage settlement, committed -baseline adoption, and publication enqueue. Adoption runs inside +baseline adoption, and publication. Adoption runs inside `Storage.commit()`: the Session passes an `adopt(seq)` callback, which the storage calls synchronously once the batch is durable and before its next commit (section 10), so memory always shows a complete prefix of the log, plus -what deferred commits hold (section 4.1). The callback keeps its own error: a failed adoption fails the Session once -`Storage.commit()` returns. A commit that rejects after its adoption fails the -Session too, and its adopted changes stay adopted. Commit observers run on the -line; watch listeners run later. -External model, process, tool, network, and human effects run outside it. -`subscribeCommits()` observes stored commits synchronously on the line after -adoption, each with exactly what that storage transaction stored (section 4.1). `subscribeClose()` observes close synchronously when it +what deferred commits hold (section 4.1). External model, process, tool, +network, and human effects run outside the line. + +The commit is published when it is adopted, so every view changes in the same +step as memory. Inside `adopt`, after the trackers, come the commit's telemetry +events (`pi.durable.entry`, `pi.durable.submission`), then the Session's own +observers, synchronously: document watches, conversation view mounts, the task +graph, the scheduler, and submission waits, with the observers adapters attach +(`observeDoc()`, `observeConversation()`, `observeTaskGraph()`, section 9.1). They +advance watches and resolve waits, but run no task code: the abort signal of a task the commit newly marks +follows in a microtask. Host `subscribeCommits()` listeners run one microtask +after `adopt`, outside the storage's synchronous step, before every wait the +commit resolves and every abort it queues: those continuations are queued by +the own observers, after the dispatch. There is no order between host listeners +and watch listeners, nor `onWake`: they are independent observers of the +commit, and a watch delivery already queued for an earlier frame may deliver +this commit first. The host listeners run before the Session sees +`Storage.commit()` resolve, while the line is still held. A storage that works +after `adopt` before it resolves, as JSONL does when it reclaims sidecars, runs +what the commit wakes during that work too; its next commit still waits. + +```text +adopt(41): trackers, telemetry events, own observers (views advance, waits resolve) +microtask: subscribeCommits() listeners +microtasks after it: wait continuations, abort listeners +microtasks, in no order relative to the above: watch listeners, onWake +Storage.commit() resolves; session.commit() resolves +``` + +`adopt` never throws. A throwing telemetry adapter is reported (`host_error`, +callback `telemetry`; a plain Session drops it, as it drops every report) and +the commit resolves; a host listener's throw is reported (section 10.1), and an +adapter's observer that throws is also detached and ended (section 9.1). The +first throw of an own observer is kept, the other observers and the host +listeners still get the commit, and it fails the Session once +`Storage.commit()` returned; the commit stands and resolves. A throw while +memory adopts the batch is different: nothing is published, the Session fails +once `Storage.commit()` returned, and the commit rejects with that error. A +commit that Storage rejects after its adoption is durable: it stays adopted and +published, waits it resolved stay resolved, the commit rejects with the storage +error, and the Session fails with it; an own observer's throw in that commit is +then reported as a `listener_error`. + +`subscribeCommits()` observes stored commits in one synchronous call per +listener, on the line, each with exactly what that storage transaction stored (section 4.1). Adoption +queues the one dispatch of a commit only when a listener is registered; the +dispatch calls the listeners registered when it runs. So a listener added +between `adopt` and the dispatch gets the commit only when some listener was +registered at `adopt`; one removed in between does not get it; one added after +the dispatch ran misses it. `subscribeClose()` observes close synchronously when it begins, after admission is sealed, also when a failure closes the Session; the Harness ends waits, watches, and streams there, and signals task invocations after every close listener has run, so what a task holds ends with the Session's reason rather than as cancelled. Both return idempotent disposers; their listeners must not block or call Session APIs, and one that throws is reported without affecting the others (section 10.1). Watch listeners -still run later, off the line. +still run later, as microtasks the line never waits for (invariant 7). ```ts await session.commit(async tx => { @@ -1834,10 +1900,13 @@ A failed Session stores nothing; after a reopen nothing is held. Publication: the Session's own observers see each held commit once, when memory adopts it, and of a normal commit that stores held changes only its own operations. `subscribeCommits()` sees every storage transaction with exactly -what it stored: nothing for a held commit, and the stored operations, held -first, for a commit or a deadline that stores them. A consumer that applies +what it stored, published when Storage adopts it (section 4): nothing for a +held commit, and the stored operations, held first, for a commit or a deadline +that stores them. A document the commit retires is published as its retirement +only: the held operations and the commit's own change stored before the +retirement are not. A consumer that applies `subscribeCommits()` publications to stored state, such as a replica, stays -equal to Storage. Its `seq` values have gaps where nothing was published. +equal to Storage in every live document. Its `seq` values have gaps where nothing was published. Consistency: every stored document is a state it had in memory, and a normal commit that reads a held document through `tx` stores it with whatever it @@ -2577,12 +2646,12 @@ paused, when the Session next needs a running process: Blocked tasks (section 5.4), `completing` tasks, and waits on live tasks count for neither; the work they wait for does. With `HarnessOptions.onWake`, the Harness arms no timer for sleeps. It calls `onWake` with `wake()` after open and -whenever the value changes, off the Session line; a throw is reported. The host +whenever the value changes, in a microtask the line never waits for; a throw is reported. The host arms its own wakeup and calls `resume()` when it fires, which makes due sleeps runnable. A host whose process can be evicted, such as a Durable Object with its one alarm, arms the alarm at `at`, or at `now + heartbeat` while `active` if that is earlier, and deletes it when neither is set. `onWake` runs after the -commit that changes the value, so a host call that can make an idle Session +commit that changes the value was adopted (section 4), so a host call that can make an idle Session active arms the recovery wakeup first: `submit()`, `compact()`, `abortTask()`, `Conversation.abort()`, a registry change, and `resume()` after open. An early wakeup is harmless; a missing one leaves work stranded. A failed alarm write @@ -4761,23 +4830,26 @@ re-exports both): ```ts type WatchEnd = - | { readonly reason: "stopped" | "cancelled" | "retired" | "ended" } + | { readonly reason: "stopped" | "cancelled" | "ended" } | { readonly reason: "listener_error"; readonly error: Error } /** The source failed; for Durable, what failed the Session, usually a storage error (section 10.1). */ | { readonly reason: "failed"; readonly error: unknown }; +/** One listener call: the newest value, the ops from the previous value, and a retirement since the previous call. */ +type WatchChange = { readonly value: T; readonly ops: readonly Op[]; readonly retired: boolean }; + interface WatchHandle { /** Acquisition revision before start; latest delivered immutable revision afterward. */ readonly value: T; /** Installs the sole serialized asynchronous listener. */ - start(listener: (value: T, ops: readonly Op[], context: Context) => Promise): void; + start(listener: (change: WatchChange, context: Context) => Promise): void; /** Idempotently stops future callbacks and returns this watch's terminal result. */ stop(): Promise; /** Settles when the watch terminates; an already-running callback remains caller-owned. */ readonly closed: Promise; } -type DocumentWatch = WatchHandle | null>; +type DocumentWatch = WatchHandle>; interface DocumentObserver { watchDoc(token: SessionDocToken, context: Context): Promise | undefined>; @@ -4790,15 +4862,33 @@ interface DocumentObserver { ``` A watch ends once, with the first reason that applies: `stopped` by `stop()`; -`cancelled` when its acquisition `Context` is cancelled; `retired` when its -document incarnation retired; `ended` when the Session closed; `listener_error` -when its listener threw or rejected; `failed` when the Session failed. +`cancelled` when its acquisition `Context` is cancelled; `ended` when the owner +of what it watches ends (the Session closed, or a task document's task became +terminal); `listener_error` when its listener threw or rejected; `failed` when +the Session failed. -Watch acquisition never creates; absent lookup returns `undefined`. Successful -acquisition binds one concrete incarnation. Retirement is an explicit event of -that incarnation, never inferred from a `null` value: it delivers a JSON `null` -root replacement `[["r", null]]` as the last frame, then ends the watch as -`retired`. A later recreation requires acquiring a new watch. +Watch acquisition never creates; absent lookup returns `undefined`. A document +watch follows its address (kind, scope, and key) from the incarnation found at +acquisition. A retirement does not end it: the next call has `retired` set, +with the last value the watch was given and no ops (Durable has no value for an +absent document; `null` never appears). A commit that changes a document and +retires it publishes the retirement only, so that call carries the value from +before the commit, not the content stored with the retirement. A document created at the address later arrives as a +root replacement `[["r", value]]` of the new incarnation, folded with what is +pending, so one call may report a retirement and carry the next incarnation: + +```text +commit: retireDoc(StateDoc); doc(StateDoc).value = 10 +call: { value: { value: 10, ... }, ops: [["r", { value: 10, ... }]], retired: true } +``` + +Task documents retire when their task becomes terminal and are never created +again, so a task document's watch gets the retirement of that commit as its last +call and then ends `ended`, once that call returned. Until then Session close or +failure still ends it at once, dropping the retirement if it was not delivered +yet, as for a watch never started. A watch of a session or conversation document that +is retired for good stays until it is stopped, its context is cancelled, or the +Session ends; it pins nothing while its address is absent. Session close ends every watch at its seal: the watch keeps its last value and receives no frame from a commit that settles during close. @@ -4816,15 +4906,72 @@ acquisition, and every later commit is buffered until `start()`. This prevents a value from being paired with operations based on a newer unseen revision. +**Observers.** Watches are for application code. Adapters that must change in +the same step as Durable's memory, such as a Chord source over a Durable view, +attach a synchronous observer instead; every watch is built on one: + +```ts +type Observer = { + advance(change: WatchChange, context: Context): void; + end(failure: { readonly error: unknown } | undefined): void; +}; +type Observation = { readonly value: T; detach(): void }; + +interface Session { + observeDoc(token: SessionDocToken, observer: Observer>, context: Context): Promise> | undefined>; + // ... and the other five scopes (section 3.3), the observer before the context +} +interface Harness { + observeConversation(id: ConversationId, observer: Observer, context: Context): Promise>; + observeTaskGraph(observer: Observer, context: Context): Promise>; +} +``` + +Attaching works like acquiring a watch, atomically on the Session line: the +observation's `value` includes every commit before it, and `advance` gets every +later change that a watch would get, unfolded, as memory adopts it: for a +stored commit inside Storage's `adopt` (section 4), for a deferred one when +memory adopts it, before it is stored (section 4.1). A commit gives a view or +the graph at most one call, and a document observer one call per change at its +address: a commit that retires the document and creates it again gives two, the +retirement and then the new incarnation's `[["r", value]]`. The task graph also +advances without a commit, when a registry change or a failed migration changes +a task's `blocked` reason (section 9.5): that call runs on the Session line, +outside any `adopt`, with a context of its own. A document observer follows its +address as a document watch does, and `observeDoc()` gives `undefined` for an +absent one. `end` comes once: when the Session closes (`undefined`) or fails +(`{ error }`), or, for a task document, right after the retirement change of +the commit that makes its task terminal, inside that commit's `adopt`. +`detach()` is idempotent, and nothing follows it, not even the rest of the +commit underway; a detached observer is referenced by nothing of the Session. +`context` governs only the attachment: cancelling it while attaching rejects +the attachment and leaves nothing attached; cancelling it afterwards does +nothing, and `detach()` ends the observation. + +The attachment completes on the line, and the caller's continuation runs +later, so a commit adopted in between reaches `advance` before the promise of +`observeXXX()` resolves. Each change carries the whole new value, and its +operations apply to the value before it, starting from `observation.value`: an +adapter that gets a change before its observation records it, and applies it +once it has the observation, or takes `change.value`. + +`advance` runs inside the storage's commit, so an observer records the change +and returns: no awaits, no Session calls, no throws, and quick, as its time is +part of every commit; so does `end` when a task ends. One that throws anyway is +reported as `listener_error`, detached, and ended with the error; the commit and +the other observers are not affected. The change object is plain data, shared by +every observer of a view and by the watches built on it, and must not be +mutated. + ### 9.2 `watchDoc` Tasks, hooks, and tools may observe any existing document for which their code has a token and owner/key. There is no additional subtree permission system inside trusted Session code. -On the Session line, `watchDoc()` resolves one existing concrete incarnation, +On the Session line, `watchDoc()` resolves the incarnation at the address, captures its current immutable tracker revision in O(1), and registers for every -later committed revision and its operation batch. It returns `undefined` when +later committed revision at that address and its operation batch. It returns `undefined` when absent. Before `start()`, `watch.value` remains the acquisition revision while later frames buffer. @@ -4833,7 +4980,7 @@ const watch = await api.watchDoc(JobOutputDoc, producerTaskId, context); if (watch === undefined) return; try { await initializeConsumer(watch.value, context); - watch.start(async (value, ops, deliveryContext) => { + watch.start(async ({ value, ops }, deliveryContext) => { await consume(value, ops, deliveryContext); }); } catch (error) { @@ -4868,10 +5015,12 @@ the trusted immutability contract. Consumers must not mutate them or retained descendants. The selected commit Context's values are preserved without inheriting the producer's cancellation. A callback owns its own asynchronous work. -A watch remains bound to its original incarnation. Retirement drops what is -pending and queues the terminal `[["r", null]]` with value `null`. After that -callback settles, the watch closes as `retired` and never -follows a replacement incarnation. +A watch follows its address (section 9.1). A retirement folds into the +pending call like any commit and sets its `retired` flag; its value is the last +one and its operations are those still pending. A later incarnation arrives as +`[["r", value]]`. A retirement that ends a task document's task is that watch's +last call: it is delivered, then the watch ends `ended` once the call returned. While its address is +absent a watch pins nothing; a new incarnation is pinned while it is watched. `stop()` is idempotent. It synchronously unregisters, discards pending frames, prevents another callback from starting, and returns the common terminal promise. @@ -4904,7 +5053,9 @@ The Harness keeps at most one mount per conversation. The first `watch()` or event attachment builds its revision on the Session line from the committed active transcript and documents; the mount is dropped when its last observer detaches. A mount derives each next revision from the Session's -commits as memory adopts them, like `watchDoc()`: it shows committed state, +commits as memory adopts them, for a stored commit inside Storage's `adopt` +(section 4) and for a held one when memory adopts it (section 4.1), like +`watchDoc()`: it shows committed state, including deferred commits before they are stored (section 4.1). One publication that touches the conversation yields one Chord batch: @@ -4929,7 +5080,9 @@ are structurally shared under the trusted immutability contract. The mount performs no semantic projection and owns no persistence. `Conversation.watch()` exposes the mount through the serialized watch of -section 9.2, with folded pending commits and a full-value overflow replacement. +section 9.2, with folded pending commits and a full-value overflow replacement; +`Harness.observeConversation()` attaches a synchronous observer of the same mount +(section 9.1). A view never retires, so its changes never set `retired`. ### 9.4 Agent events (experimental) @@ -5014,7 +5167,8 @@ type MessageChange = interface AgentEventStream { /** The `snapshot` event at attachment. */ readonly snapshot: Extract; - start(listener: (events: readonly AgentEvent[], context: Context) => Promise): void; + /** Each call's `value` is one commit's batch, or a `snapshot` after an overflow (Chord's `NativeWatchChange`). */ + start(listener: (change: { readonly value: readonly AgentEvent[] }, context: Context) => Promise): void; stop(): Promise; readonly closed: Promise; } @@ -5144,6 +5298,7 @@ type TaskGraphWatch = WatchHandle; interface Harness { watchTaskGraph(context: Context): Promise; + observeTaskGraph(observer: Observer, context: Context): Promise>; } ``` @@ -5170,19 +5325,25 @@ A UI that places it under its parent reads the conversation's owner edge from The Harness keeps at most one graph mount. The first `watchTaskGraph()` builds its revision on the Session line from the committed live tasks and the conversations they own, one owner scan per live task, and holds the line while it does; the mount is dropped when its last observer -detaches. It advances from `subscribeCommits()` publications like the -conversation view: one publication that changes a node yields one Chord batch +detaches. It advances from the Session's stored commits as memory adopts them, +inside Storage's `adopt` (section 4), like the conversation view: a commit that +touches a conversation view and the graph changes both in that one step. A +`blocked` reason that changes without a commit (a registry change, a failed +migration) is derived again in a job on the Session line and published there, +outside any `adopt` and any commit, with the background context. One +publication that changes a node yields one Chord batch that sets or deletes `tasks[id]`, or sets a node's `conversations`. A publication that changes no node creates no revision. Task and conversation records are immutable values; a node shares nothing with them. `watchTaskGraph()` returns the mount through the serialized watch of section -9.2. It reads only: it never enables scheduling. Close ends it at its +9.2, and `observeTaskGraph()` attaches a synchronous observer of it (section +9.1). Both read only: they never enable scheduling. Close ends them at its seal like every other watch (section 2.2). ```ts const graph = await harness.watchTaskGraph(context); -graph.start(async (value) => { +graph.start(async ({ value }) => { for (const node of Object.values(value.tasks)) { const where = node.owner === undefined ? `conversation ${node.conversationId}` : `task ${node.owner}`; render(`${node.id} ${node.kind} ${node.state.status} under ${where}`); @@ -5292,7 +5453,8 @@ interface Storage { /** * Persist the batch atomically. Once it is durable, call `adopt(seq)` exactly once, synchronously and before this * storage's next commit starts, then resolve with `seq`; a commit that fails before the batch is durable never calls - * it. `adopt` must not throw. + * it. `adopt` adopts and publishes the batch: it runs the Session's own observers (section 4), which call no Storage + * method, and it never throws. */ commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise; /** @@ -5437,8 +5599,12 @@ exempt: once a batch is admitted, whether it committed is unknown. A commit callback that catches a failed read cannot commit afterwards: the Session checks its health after the callback. Adopting a committed batch in memory that fails also fails the Session, since memory would be behind storage; so does a -`Storage.commit()` that resolves without adopting, or rejects after it adopted -(the adopted changes are not rolled back). +`Storage.commit()` that resolves without adopting, or rejects after it adopted. +A batch Storage adopted is durable, so it is not rolled back: it was published +at adoption (section 4), `subscribeCommits()` listeners get it and the waits it +settles resolve; then the commit rejects with the storage error and the Session +fails as above, with the storage error. An own observer that threw on that +commit is reported as a `listener_error`. **The Session's own bookkeeping fails it too.** A throw in a commit the scheduler makes for itself (reconciliation, reservation, a fault or terminal @@ -5448,7 +5614,9 @@ storage failure, a host callback such as a `RegistryReader` whose `snapshot()` throws, or a bug, and running the commit again fixes none of them. A throw in a commit listener of the Session's own components (scheduler, submissions, views, task graph, document observers) fails it as well, since their memory would fall -behind storage; the commit itself stands. +behind storage; the commit itself stands. Those listeners run inside Storage's +`adopt` (section 4), so the Session fails once `Storage.commit()` returned, +after the other observers and the `subscribeCommits()` listeners got the commit. **User code fails only its unit.** A task phase that throws ends that task `faulted`; a throwing `migrate` blocks it as `migration_failed`; a tool that @@ -5461,8 +5629,12 @@ Callbacks that must not fail their caller are contained: a host view observer that throws, or returns a promise that rejects, are reported, and the others still run; a watch whose listener throws ends with `listener_error`. A throwing or rejecting `onReport` is -dropped, since nothing is left to report it to. A throwing `now()` is reported -once, and `Date.now` serves instead. +dropped, since nothing is left to report it to, and so is a report's own +metric or event that throws. A throwing `now()` is reported once, and +`Date.now` serves instead. A telemetry adapter that throws on a commit's events +or on the active-task metric is reported as `host_error` with callback +`telemetry` (a plain Session drops it); the commit resolves, and memory and the +views keep it. **Every report says what happened.** `onReport(error, info)` gets a `ReportInfo` whose `type` is one of: @@ -5482,7 +5654,8 @@ once, and `Date.now` serves instead. outside a task the conversation whose agent was resolved; - `host_error`: a host callback, by its `HarnessOptions` name: `now`, `settings`, `registry`, `env`, `models` for provider calls such as cancelling - a deferred response, `setTimer`, or `onWake`; with the task that called it, + a deferred response, `setTimer`, `onWake`, or `telemetry` for a telemetry + adapter that threw while a commit was published; with the task that called it, if one did; - `listener_error`: a commit, close, watch, view, or document listener. diff --git a/packages/durable/src/env/index.ts b/packages/durable/src/env/index.ts index 0d8da86f7..170c43579 100644 --- a/packages/durable/src/env/index.ts +++ b/packages/durable/src/env/index.ts @@ -124,7 +124,7 @@ export interface WatchTarget { * - `error`: the watcher stopped, for example because the watched tree grew past the environment's limit; no calls * follow. */ -export type WatchChange = { paths: string[] } | { overflow: true } | { error: FileError }; +export type FileChange = { paths: string[] } | { overflow: true } | { error: FileError }; export interface FileWatcher { /** @@ -257,11 +257,11 @@ export interface FileSystem { /** * Report changes to files and directories, for hosts that load resources from the environment. When the returned * watcher exists, coverage is established: a host that watches before it loads cannot miss a change made during the - * load. See `WatchChange` for what is reported and `FileWatcher.mode` for how reliably. + * load. See `FileChange` for what is reported and `FileWatcher.mode` for how reliably. */ watch( targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, context: Context, ): Promise>; canonicalPath(path: string, context: Context): Promise>; diff --git a/packages/durable/src/env/node-watch.ts b/packages/durable/src/env/node-watch.ts index 7f3b8ab55..84d635e6f 100644 --- a/packages/durable/src/env/node-watch.ts +++ b/packages/durable/src/env/node-watch.ts @@ -3,7 +3,7 @@ import { type FSWatcher, watch as fsWatch, type Stats } from "node:fs"; import { lstat, readdir, readFile, stat, statfs } from "node:fs/promises"; import { dirname, relative, sep } from "node:path"; import type { Context } from "@earendil-works/chord"; -import { FileError, type FileWatcher, type WatchChange, type WatchTarget } from "./index.ts"; +import { type FileChange, FileError, type FileWatcher, type WatchTarget } from "./index.ts"; /** How `NodeExecutionEnv` watches. */ export interface NodeWatchOptions { @@ -138,7 +138,7 @@ function excluded(target: ResolvedTarget, name: string): boolean { */ export class NodeFileWatcher implements FileWatcher { readonly #targets: readonly ResolvedTarget[]; - readonly #onChange: (change: WatchChange) => void; + readonly #onChange: (change: FileChange) => void; readonly #pollIntervalMs: number; readonly #maxDirectories: number; readonly #watchers = new Map(); @@ -153,7 +153,7 @@ export class NodeFileWatcher implements FileWatcher { private constructor( targets: readonly ResolvedTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, mode: "native" | "polling", options: NodeWatchOptions, ) { @@ -168,7 +168,7 @@ export class NodeFileWatcher implements FileWatcher { static async open( targets: readonly WatchTarget[], resolvePath: (path: string) => string, - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, options: NodeWatchOptions, ): Promise { const resolved = targets.map((target) => ({ @@ -216,7 +216,7 @@ export class NodeFileWatcher implements FileWatcher { this.#watchers.clear(); } - #deliver(change: WatchChange): void { + #deliver(change: FileChange): void { if (this.#closed) return; try { this.#onChange(change); diff --git a/packages/durable/src/env/node.ts b/packages/durable/src/env/node.ts index fe93ce4ed..d45ad2cae 100644 --- a/packages/durable/src/env/node.ts +++ b/packages/durable/src/env/node.ts @@ -28,6 +28,7 @@ import { type ExecutionEnv, ExecutionError, err, + type FileChange, FileError, type FileInfo, type FileKind, @@ -41,7 +42,6 @@ import { type TextLine, type TextLineReader, toError, - type WatchChange, type WatchTarget, } from "./index.ts"; import { LineScanner } from "./line-scan.ts"; @@ -627,7 +627,7 @@ export class NodeExecutionEnv implements ExecutionEnv { async watch( targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, context: Context, ): Promise> { const aborted = abortResult(context.abortSignal); diff --git a/packages/durable/src/harness/events.ts b/packages/durable/src/harness/events.ts index ae3e4de9d..f34af4659 100644 --- a/packages/durable/src/harness/events.ts +++ b/packages/durable/src/harness/events.ts @@ -1,4 +1,10 @@ -import { type Context, type JsonValue, NativeWatch, type WatchEnd } from "@earendil-works/chord"; +import { + type Context, + type JsonValue, + NativeWatch, + type NativeWatchChange, + type WatchEnd, +} from "@earendil-works/chord"; import type { Op, Path } from "@earendil-works/chord/delta"; import type { AssistantMessage, Message, Usage } from "@earendil-works/pi-ai"; import type { ObservedCommit } from "../session/session.ts"; @@ -114,7 +120,8 @@ export type ToolEventCall = { export interface AgentEventStream { /** The `snapshot` event at attachment. */ readonly snapshot: SnapshotEvent; - start(listener: (events: readonly AgentEvent[], context: Context) => Promise): void; + /** Install the sole listener; each call's `value` is one commit's batch, or a `snapshot` after an overflow. */ + start(listener: (change: NativeWatchChange, context: Context) => Promise): void; stop(): Promise; readonly closed: Promise; } diff --git a/packages/durable/src/harness/harness.ts b/packages/durable/src/harness/harness.ts index 7ce8de9c8..1df3422d7 100644 --- a/packages/durable/src/harness/harness.ts +++ b/packages/durable/src/harness/harness.ts @@ -28,6 +28,8 @@ import type { EntryId, EntryQuery, EntryRecord, + Observation, + Observer, Page, ReportInfo, Storage, @@ -46,7 +48,7 @@ import { ProviderDoc } from "./provider.ts"; import { BUILTIN_TASKS } from "./registry.ts"; import { type InvocationBinding, TaskScheduler } from "./scheduler.ts"; import { Submissions } from "./submissions.ts"; -import { TaskGraphView, type TaskGraphWatch } from "./task-graph.ts"; +import { type TaskGraph, TaskGraphView, type TaskGraphWatch } from "./task-graph.ts"; import { DURABLE_TELEMETRY_SCHEMA } from "./telemetry.ts"; import type { Agent, @@ -72,7 +74,7 @@ import type { } from "./types.ts"; import { addUsageState, UsageDoc, type UsageState } from "./usage.ts"; import { scanAll } from "./util.ts"; -import { ConversationViews } from "./view.ts"; +import { type ConversationView, ConversationViews } from "./view.ts"; const SCAN_PAGE_SIZE = 256; @@ -373,6 +375,20 @@ class HarnessImpl extends SessionImpl implements return this.#taskGraph.watch(context); } + async observeTaskGraph(observer: Observer, context: Context): Promise> { + this.assertUsable(); + return this.#taskGraph.observe(observer, context); + } + + async observeConversation( + id: ConversationId, + observer: Observer, + context: Context, + ): Promise> { + this.assertUsable(); + return this.#host.views.observe(id, observer, context); + } + root( context: Context, options?: { readonly agent?: AgentChange; readonly init?: ConversationInit }, @@ -433,7 +449,10 @@ class HarnessImpl extends SessionImpl implements } } -/** Preserve the host callback's error containment; telemetry adapters are responsible for their own failures. */ +/** + * Report to `onReport`, then count it and record its event. Never throws, since reports also come from inside Storage's + * `adopt`: a throwing `onReport` is dropped, and so is a throwing metric or event, as reporting it would report again. + */ function safeReport( onReport: ((error: unknown, info: ReportInfo) => void) | undefined, telemetry: TelemetryContext, @@ -467,8 +486,12 @@ function safeReport( attributes["task.id"] = info.task?.id; break; } - reports.add(1, { type: info.type }); - context?.telemetry.addEvent("pi.durable.report", attributes); + try { + reports.add(1, { type: info.type }); + context?.telemetry.addEvent("pi.durable.report", attributes); + } catch { + // Dropped: see above. + } }; } diff --git a/packages/durable/src/harness/scheduler.ts b/packages/durable/src/harness/scheduler.ts index 9a7fcf2e2..0840533a8 100644 --- a/packages/durable/src/harness/scheduler.ts +++ b/packages/durable/src/harness/scheduler.ts @@ -664,9 +664,10 @@ export class TaskScheduler { } if (record.abortRequested && previous?.abortRequested !== true) { this.#cascadePending = true; - // Signal a run invocation of the newly marked task; its next step ends it. + // Signal a run invocation of the newly marked task; its next step ends it. Abort listeners are task code, which + // never runs inside Storage's `adopt`: the signal follows in a microtask, after host commit listeners. const invocation = this.#invocations.get(record.id); - if (invocation?.mode === "run") invocation.controller.abort(); + if (invocation?.mode === "run") queueMicrotask(() => invocation.controller.abort()); } // A request replacing a `restart` mark upgrades the marks of the work below. if (previous?.abortReason !== undefined && record.abortReason === undefined) this.#cascadePending = true; @@ -1023,10 +1024,10 @@ export class TaskScheduler { const invocation = this.#invocations.get(id); if (invocation !== undefined) invocation.record = record as TaskRecord; if (record.state.status === "terminal") { - if (previous !== undefined && !this.#closing) this.#activeMetric.add(-1, { task: record.kind }); + if (previous !== undefined && !this.#closing) this.#countActive(-1, record.kind); this.#resumed.delete(id); } else if (previous === undefined && !this.#closing) { - this.#activeMetric.add(1, { task: record.kind }); + this.#countActive(1, record.kind); } if (previous?.state.status === "waiting") for (const on of previous.state.on) deleteFrom(this.#waiters, on, id); if (record.state.status === "terminal") { @@ -1304,6 +1305,18 @@ export class TaskScheduler { return undefined; } + /** + * Count a task in or out of `pi.durable.tasks.active`. Commit observers count inside Storage's `adopt`, which must not + * throw, so a throwing adapter is reported. + */ + #countActive(delta: 1 | -1, kind: string): void { + try { + this.#activeMetric.add(delta, { task: kind }); + } catch (error) { + this.#report(error, TELEMETRY); + } + } + /** Fail the Session, which reports it, for a throw in one of the scheduler's own commits; ignored once closing. */ #failSession(error: unknown, context?: Context): void { if (!this.#closing) this.#session.fail(error, context); @@ -1311,7 +1324,7 @@ export class TaskScheduler { /** Close listener: runs synchronously once admission is sealed, before `join()`, or when the Session fails. */ #seal(): void { - for (const record of this.#live.values()) this.#activeMetric.add(-1, { task: record.kind }); + for (const record of this.#live.values()) this.#countActive(-1, record.kind); this.#resumed.clear(); this.#closing = true; try { @@ -1329,7 +1342,10 @@ export class TaskScheduler { this.#timer = undefined; } - /** Recompute the wake in a microtask, off the Session line: arm the built-in timer, or tell `onWake` of a change. */ + /** + * Recompute the wake in a microtask the line never waits for, with no order relative to host commit listeners (one + * already queued serves later changes too): arm the built-in timer, or tell `onWake` of a change. + */ #scheduleWake(): void { if (this.#wakeScheduled || this.#closing || this.#readOnly) return; this.#wakeScheduled = true; @@ -2385,6 +2401,7 @@ export class TaskScheduler { const REGISTRY: ReportInfo = { type: "host_error", callback: "registry" }; const SETTINGS: ReportInfo = { type: "host_error", callback: "settings" }; const ON_WAKE: ReportInfo = { type: "host_error", callback: "onWake" }; +const TELEMETRY: ReportInfo = { type: "host_error", callback: "telemetry" }; /** The extension that installed `task` in `snapshot`, the last one that lists it; none for a built-in task. */ function installerOf(snapshot: RegistrySnapshot, task: AnyTask | undefined): string | undefined { diff --git a/packages/durable/src/harness/task-graph.ts b/packages/durable/src/harness/task-graph.ts index 55ee659ed..828e7077e 100644 --- a/packages/durable/src/harness/task-graph.ts +++ b/packages/durable/src/harness/task-graph.ts @@ -1,8 +1,17 @@ -import { type Context, type JsonValue, Watch, type WatchHandle } from "@earendil-works/chord"; +import { type Context, type JsonValue, Watch, type WatchChange, type WatchHandle } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; -import { contained, type ObservedCommit, type SessionImpl } from "../session/session.ts"; -import type { ConversationId, JoinPolicy, Storage, TaskId, TaskOutcome, TaskRecord } from "../types.ts"; +import { contained, guardObserver, type ObservedCommit, observation, type SessionImpl } from "../session/session.ts"; +import type { + ConversationId, + JoinPolicy, + Observation, + Observer, + Storage, + TaskId, + TaskOutcome, + TaskRecord, +} from "../types.ts"; import { type BlockedReason, type BlockedSubject, blockedSubject } from "./scheduler.ts"; import { closedError, scanAll } from "./util.ts"; @@ -47,10 +56,10 @@ export type TaskGraph = { export type TaskGraphWatch = WatchHandle; type AnyTaskRecord = TaskRecord; -type Observer = Watch; type Mount = { value: TaskGraph; - readonly observers: Set; + /** Watches and guarded host observers (`observeTaskGraph()`). */ + readonly observers: Set>; /** What the blocked reason reads of each live task, so a registry change can derive it again without payloads. */ readonly records: Map; }; @@ -97,7 +106,25 @@ export class TaskGraphView { }); } - /** A serialized watch of the graph; cancelling `context` stops it. */ + /** + * Attach a synchronous observer of the graph (`Harness.observeTaskGraph()`), behind `guardObserver()`; `context` + * governs only the attachment. + */ + async observe(observer: Observer, context: Context): Promise> { + let value!: TaskGraph; + const attached = await this.#attach((current, release) => { + value = current; + return guardObserver(observer, release, (error) => this.#session.report(error)); + }, context); + const signal = context.abortSignal; + if (signal?.aborted) { + attached.detach(); + throw signal.reason; + } + return observation(value, attached.observer); + } + + /** A serialized watch of the graph, an observer of its mount; cancelling `context` stops it. */ async watch(context: Context): Promise { const { observer } = await this.#attach( (value, release) => new Watch(value, release, (error) => this.#session.report(error)), @@ -113,12 +140,12 @@ export class TaskGraphView { } /** Register an observer created from the current revision, atomically on the Session line. */ - #attach( + #attach>( create: (value: TaskGraph, release: () => void) => O, context: Context, ): Promise<{ observer: O; detach: () => void }> { return this.#session.readOnLine(async () => { - const mount = this.#mount ?? { ...(await this.#build(context)), observers: new Set() }; + const mount = this.#mount ?? { ...(await this.#build(context)), observers: new Set>() }; const detach = (): void => { mount.observers.delete(observer); if (mount.observers.size === 0 && this.#mount === mount) this.#mount = undefined; @@ -232,8 +259,8 @@ function publish(mount: Mount, ops: Op[], context: Context, report: (error: unkn if (ops.length === 0) return; mount.value = applyImmutable(mount.value, ops); const frameContext = withoutAbortSignal(context); - for (const observer of [...mount.observers]) - contained(() => observer.advance(mount.value, ops, frameContext), report); + const change: WatchChange = { value: mount.value, ops, retired: false }; + for (const observer of [...mount.observers]) contained(() => observer.advance(change, frameContext), report); } function nodeOf( diff --git a/packages/durable/src/harness/types.ts b/packages/durable/src/harness/types.ts index 748696f8c..61a9c0ca6 100644 --- a/packages/durable/src/harness/types.ts +++ b/packages/durable/src/harness/types.ts @@ -30,6 +30,8 @@ import type { EntryQuery, EntryRecord, JsonObject, + Observation, + Observer, Page, ReportInfo, Session, @@ -43,7 +45,7 @@ import type { TaskState, Tx, } from "../types.ts"; -import type { TaskGraphWatch } from "./task-graph.ts"; +import type { TaskGraph, TaskGraphWatch } from "./task-graph.ts"; import type { UsageState } from "./usage.ts"; import type { ConversationView } from "./view.ts"; @@ -661,9 +663,10 @@ export type HarnessOptions = { * Wakes the process for sleeping tasks instead of a `setTimer` timer. Without it, while scheduling runs, a timer that * keeps the process alive resumes sleeping tasks when they are due. With it, the Harness arms no timer for them: it * calls `onWake` after open and whenever `Harness.wake()` changes, and the host arms its own wakeup, such as a - * Durable Object alarm, and calls `resume()` when it fires. Called off the Session line, after the commit that - * changed the value, so a host arms a recovery wakeup before calls that can start work. It may call `wake()` and - * `resume()`, and must not block. A throw is reported, and the value is offered again at the next change. + * Durable Object alarm, and calls `resume()` when it fires. Called in a microtask the Session line never waits for, + * after the commit that changed the value was adopted, so a host arms a recovery wakeup before calls that can start + * work. It may call `wake()` and `resume()`, and must not block. A throw is reported, and the value is offered again + * at the next change. */ readonly onWake?: (wake: Wake) => void; /** @@ -791,7 +794,10 @@ export interface Conversation { * owned, transitively, by its non-background tasks. */ waitForIdle(context: Context): Promise; - /** The structural view (spec §9.3) as a serialized watch; commits made while its listener runs arrive folded. */ + /** + * The structural view (spec §9.3) as a serialized watch; commits made while its listener runs arrive folded. Built on + * the same mount as `Harness.observeConversation()`. + */ watch(context: Context): Promise; } @@ -854,6 +860,23 @@ export interface Harness extends Session { * made while its listener runs arrive folded. */ watchTaskGraph(context: Context): Promise; + /** + * Attach a synchronous `Observer` to a conversation's structural view (spec §9.3), atomically on the Session line, + * for adapters: `advance` gets each next revision with its operations as memory adopts the commit, inside Storage's + * `adopt` for a stored one and before it is stored for a deferred one, so it only records it. `context` governs only + * the attachment. + */ + observeConversation( + id: ConversationId, + observer: Observer, + context: Context, + ): Promise>; + /** + * Attach a synchronous `Observer` to the task graph (spec §9.5), as `observeConversation()` does to a view. A + * `blocked` reason that changes without a commit (a registry change, a failed migration) advances it on the Session + * line outside any commit and `adopt`, with the background context. + */ + observeTaskGraph(observer: Observer, context: Context): Promise>; } /** What a hook may use: committed reads and the asking task's memos, which hooks and the task share. */ diff --git a/packages/durable/src/harness/view.ts b/packages/durable/src/harness/view.ts index 7f6f1a408..d12632b24 100644 --- a/packages/durable/src/harness/view.ts +++ b/packages/durable/src/harness/view.ts @@ -1,7 +1,7 @@ -import { type Context, type JsonValue, Watch, type WatchHandle } from "@earendil-works/chord"; +import { type Context, type JsonValue, Watch, type WatchChange, type WatchHandle } from "@earendil-works/chord"; import { withoutAbortSignal } from "@earendil-works/chord/context"; import { applyImmutable, type NonEmptyPath, type Op, type Path } from "@earendil-works/chord/delta"; -import { contained, type ObservedCommit, type SessionImpl } from "../session/session.ts"; +import { contained, guardObserver, type ObservedCommit, observation, type SessionImpl } from "../session/session.ts"; import type { ConversationDocToken, ConversationId, @@ -9,6 +9,8 @@ import type { DocumentId, EntryRecord, JsonObject, + Observation, + Observer, Storage, } from "../types.ts"; import { AgentDoc } from "./agent.ts"; @@ -30,7 +32,8 @@ export type ConversationView = { /** Receives each next revision of a mount, and the Session's close. */ export type ViewObserver = { - advance?(value: ConversationView, ops: readonly Op[], context: Context): void; + /** Each next revision; one change object is shared by every observer of the mount. */ + advance?(change: WatchChange, context: Context): void; /** Every publication, after the mount took it; `ops` are the mount's, possibly none. */ publication?( before: ConversationView, @@ -106,7 +109,33 @@ export class ConversationViews { this.#session.report(error); } - /** A serialized watch of the view; cancelling `context` stops it. */ + /** + * Attach a synchronous observer of the view (`Harness.observeConversation()`), behind `guardObserver()`; `context` + * governs only the attachment. + */ + async observe( + id: ConversationId, + observer: Observer, + context: Context, + ): Promise> { + let value!: ConversationView; + const attached = await this.attach( + id, + (current, release) => { + value = current; + return guardObserver(observer, release, (error) => this.report(error)); + }, + context, + ); + const signal = context.abortSignal; + if (signal?.aborted) { + attached.detach(); + throw signal.reason; + } + return observation(value, attached.observer); + } + + /** A serialized watch of the view, an observer of its mount; cancelling `context` stops it. */ async watch(id: ConversationId, context: Context): Promise> { const { observer } = await this.attach( id, @@ -219,8 +248,9 @@ function advance( if (ops.length > 0) { const value = docOps.length === 0 ? mount.value : applyImmutable(mount.value, docOps); mount.value = entries === value.entries ? value : { ...value, entries }; + const change: WatchChange = { value: mount.value, ops, retired: false }; for (const observer of [...mount.observers]) { - contained(() => observer.advance?.(mount.value, ops, frameContext), report); + contained(() => observer.advance?.(change, frameContext), report); } } for (const observer of [...mount.observers]) { diff --git a/packages/durable/src/index.ts b/packages/durable/src/index.ts index b26a50316..3baf2e684 100644 --- a/packages/durable/src/index.ts +++ b/packages/durable/src/index.ts @@ -1,4 +1,4 @@ -export type { WatchEnd, WatchHandle } from "@earendil-works/chord"; +export type { NativeWatchChange, WatchChange, WatchEnd, WatchHandle } from "@earendil-works/chord"; export { defineDoc, defineDocFamily } from "./documents.ts"; export { AssistantEntry, @@ -171,6 +171,8 @@ export type { JsonObject, LatestConversationSemantics, NextTaskState, + Observation, + Observer, Page, PhaseHandler, ReportFrom, diff --git a/packages/durable/src/session/session.ts b/packages/durable/src/session/session.ts index d66045b19..1c0f3fd0c 100644 --- a/packages/durable/src/session/session.ts +++ b/packages/durable/src/session/session.ts @@ -1,6 +1,7 @@ import { type Context, type JsonValue, Watch } from "@earendil-works/chord"; import { awaitWithContext, BACKGROUND_CONTEXT, withoutAbortSignal } from "@earendil-works/chord/context"; import { type Op, PendingOps, track } from "@earendil-works/chord/delta"; +import type { SpanAttributes } from "@earendil-works/pi-telemetry"; import { type AnyDocToken, addressId, @@ -35,6 +36,8 @@ import type { EntryRecord, Id, JsonObject, + Observation, + Observer, Page, ReportInfo, RewindableConversationDocFamilyToken, @@ -68,9 +71,6 @@ import { type TransactionScope, } from "./transaction.ts"; -/** What a document watch observes: the committed value of its incarnation, `null` once it retired. */ -type ObservedDocumentValue = Readonly | null; - /** * What the Session's own components observe of one commit (`observeCommits()`): a stored commit with its own changes, or * a deferred one adopted in memory only (`CommitPersistence`), which changes documents only. Memory is never behind: @@ -94,9 +94,18 @@ const NO_HOLDS: readonly Hold[] = []; const NO_OPERATIONS: readonly Op[] = []; +/** + * Queues a commit's host dispatch as one microtask through `then`: in the same queue as `queueMicrotask()`, which Node + * wraps in an async resource that costs about twice the memory and three times the time per stored commit. + */ +const SETTLED: Promise = Promise.resolve(); + /** The default source of a report: a listener, which never fails what ran it. */ const LISTENER: ReportInfo = { type: "listener_error" }; +/** A telemetry adapter that threw on a commit's event, which must not fail the durable commit. */ +const TELEMETRY: ReportInfo = { type: "host_error", callback: "telemetry" }; + /** * Open a Session kernel over one storage backend. `now` is the wall clock for task times; default `Date.now`. * `setTimer` arms the timers that evict idle documents; default `setTimeout` (see `HarnessOptions.setTimer`). The first @@ -159,10 +168,12 @@ export function persistWithin(ms: number | undefined): number { * line, so no transaction ever holds a tracker that leaves. * * Only committed state is observable. A deferred commit (`CommitPersistence`) is committed in memory first: it is - * adopted and shown to the Session's own observers, and its changes wait, folded per document and pinned in the cache, for - * the first commit that reads or writes the document, the document's deadline, a drop of its tracker, or close, which - * store them. Every commit callback, preparation, Storage settlement, adoption, and publication enqueue runs while the - * line is held; listeners run later. + * adopted and shown to the Session's own observers, and its changes wait, folded per document and pinned in the cache, + * for the first commit that reads or writes the document, the document's deadline, a drop of its tracker, or close, + * which store them. Every commit callback, preparation, Storage settlement, and adoption runs while the line is held. A + * stored commit is published when Storage adopts it: inside `adopt` its telemetry events and the Session's own + * observers, one microtask later `subscribeCommits()` listeners, before every wait the commit resolves and every abort + * it queues, all still on the line. Host listeners and watch listeners have no order. * * Storage failures are final. Every Storage call goes through one guard: the first error a Storage method throws, other * than for a cancelled caller, fails the Session, as does a commit it cannot adopt. That call gets the error; every later @@ -437,35 +448,92 @@ export class SessionImpl implements Session { key: string, context: Context, ): Promise | undefined>; - async watchDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise | undefined> { - this.assertUsable(); - const definition = token.definition; - const resolved = resolveAddress(definition, args); - const context = args[resolved.nextArgument] as Context; - const signal = context.abortSignal; - let cancelled = signal?.aborted ?? false; - const markCancelled = (): void => { - cancelled = true; - }; - signal?.addEventListener("abort", markCancelled, { once: true }); + watchDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise | undefined> { + // Not async, so an attachment costs one async frame, `#attachDocument`'s; a throw here still rejects. try { - const watch = await this.#enqueue(async () => { - this.#assertHealthy(); - if (cancelled) throw cancellationError(signal!); - const loaded = await this.#loadDocument(definition, resolved.id, resolved.address, context); - if (cancelled) throw cancellationError(signal!); - if (loaded === undefined) return undefined; - return this.#attachDocument(definition, loaded); - }); - if (watch === undefined) return undefined; - if (cancelled) { - watch.cancel(); - throw cancellationError(signal!); - } - if (signal !== undefined) watch.observeCancellation(signal); - return watch as DocumentWatch; - } finally { - signal?.removeEventListener("abort", markCancelled); + this.assertUsable(); + const definition = token.definition; + const resolved = resolveAddress(definition, args); + const context = args[resolved.nextArgument] as Context; + const signal = context.abortSignal; + return this.#attachDocument( + definition, + resolved, + context, + (loaded) => { + const watch = this.#observeDocument(definition, loaded, createWatch, endWatch); + if (signal !== undefined) watch.observeCancellation(signal); + return watch; + }, + cancelWatch, + ); + } catch (error) { + return Promise.reject(error); + } + } + + observeDoc( + token: SessionDocToken, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocToken, + conversationId: ConversationId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocToken, + taskId: TaskId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: SessionDocFamilyToken, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocFamilyToken, + conversationId: ConversationId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocFamilyToken, + taskId: TaskId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc(token: AnyDocToken, ...args: readonly unknown[]): Promise> | undefined> { + // Not async, as `watchDoc()`; a throw here still rejects. + try { + this.assertUsable(); + const definition = token.definition; + const resolved = resolveAddress(definition, args); + const observer = args[resolved.nextArgument] as Observer>; + const context = args[resolved.nextArgument + 1] as Context; + return this.#attachDocument( + definition, + resolved, + context, + (loaded) => { + const guarded = this.#observeDocument( + definition, + loaded, + (_value, detach, report) => guardObserver(observer, detach, report), + endGuarded, + ); + return observation(loaded.tracker.value, guarded); + }, + detachObservation, + ); + } catch (error) { + return Promise.reject(error); } } @@ -607,8 +675,10 @@ export class SessionImpl implements Session { } /** - * Register a synchronous listener for every stored commit, with exactly what it stored (`CommitPublication`). It must - * not block or call Session operations; a throw is reported. + * Register a synchronous listener for every stored commit, with exactly what it stored (`CommitPublication`). It runs + * one microtask after Storage adopted the commit, on the line, before every wait the commit resolves and every abort + * it queues (in no order with watch listeners); it must not block or call Session operations, and a throw is + * reported. */ subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void { this.assertUsable(); @@ -618,8 +688,9 @@ export class SessionImpl implements Session { /** * Internal: observe every commit as memory adopts it (`ObservedCommit`), for the Session's own components, whose state - * follows memory. A throw leaves that state behind, so it fails the Session instead of being reported. They run - * before host listeners. + * follows memory. A throw leaves that state behind, so it fails the Session instead of being reported. For a stored + * commit they run inside Storage's `adopt`, so they call no Storage method, run no task code, and contain every host + * callback; the waits they resolve and the aborts they queue run after host listeners. */ observeCommits(listener: (publication: ObservedCommit, context: Context) => void): () => void { this.assertUsable(); @@ -805,23 +876,35 @@ export class SessionImpl implements Session { } // The last step before Storage: held changes leave their queue only once nothing but Storage can fail. const taken = this.#holds.size === 0 ? NO_HOLDS : tx.takeHeld((id) => this.#holds.get(id)); - // Storage adopts the batch synchronously once it is durable, before its next commit; a throw here would reach - // Storage, so the callback keeps its own error. - let adoption: { readonly documents: AdoptedDocuments } | { readonly error: unknown } | undefined; + // Storage adopts the batch synchronously once it is durable, before its next commit, and the commit is published in + // the same step, so every view changes with memory. A throw here would reach Storage, so the callback keeps its own + // error, and the first error of an own observer. + let adoption: + | { readonly documents: AdoptedDocuments; readonly failure?: { readonly error: unknown } } + | { readonly error: unknown } + | undefined; const adopt = (committed: Seq): void => { + let documents: AdoptedDocuments; try { - adoption = { documents: tx.adopt(committed) }; + documents = tx.adopt(committed); } catch (error) { adoption = { error }; + return; } + const failure = this.#publish(committed, writes, documents, context); + adoption = failure === undefined ? { documents } : { documents, failure }; }; - let seq: Seq; try { // Once admitted, caller cancellation does not interrupt Storage settlement. - seq = await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); + await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); } catch (error) { - // The guard has failed the Session. A rejection after adoption leaves nothing staged to discard. + // The guard has failed the Session with the storage error. A rejection after adoption leaves nothing staged to + // discard, and the commit, durable, stays published; an own observer's error that would have failed the Session + // is still reported. if (adoption === undefined) tx.discard(); + else if ("documents" in adoption && adoption.failure !== undefined) { + this.report(adoption.failure.error, LISTENER, context); + } throw error; } if (adoption === undefined) { @@ -834,10 +917,10 @@ export class SessionImpl implements Session { this.fail(adoption.error); throw adoption.error; } - const documents = adoption.documents; if (taken.length > 0) this.#release(taken); - for (const document of documents.stored) this.#touchDocument(addressId(document.record)); - this.#publish(seq, writes, documents, context); + for (const document of adoption.documents.stored) this.#touchDocument(addressId(document.record)); + // The commit is durable and published, so it stands; memory behind an own observer's state fails the Session. + if (adoption.failure !== undefined) this.fail(adoption.failure.error, context); this.#sweepDocuments(); return result; } @@ -869,16 +952,19 @@ export class SessionImpl implements Session { this.#armHoldTimer(); const publication: ObservedCommit = { kind: "memory", changes: documents }; for (const listener of [...this.#internalListeners]) { - contained( - () => listener(publication, context), - (error) => this.fail(error, context), - ); + // Our own observers are synchronous. + try { + listener(publication, context); + } catch (error) { + this.fail(error, context); + } } } /** * Store the held changes of `holds` in one storage transaction, on the line. Memory already has them, so only - * `subscribeCommits()` listeners see it. A failure fails the Session, as every Storage failure does. + * `subscribeCommits()` listeners see it, published as Storage adopts it. A failure fails the Session, as every Storage + * failure does. */ async #storeHolds(holds: readonly Hold[], context: Context): Promise { if (holds.length === 0) return; @@ -894,27 +980,21 @@ export class SessionImpl implements Session { changes.push({ type: "document", record, conversationId, version, value, ops: stored.ops }); } let adopted = false; - const adopt = (): void => { + const adopt = (seq: Seq): void => { adopted = true; for (const [index, hold] of holds.entries()) { if (writes[index]!.content.kind === "base") hold.loaded.deltasSinceBase = 0; else hold.loaded.deltasSinceBase++; } + if (this.#commitListeners.size > 0) this.#notifyHosts({ seq, changes }, context); }; - const seq = await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); + await this.#storage.commit(writes, adopt, withoutAbortSignal(context)); if (!adopted) { const error = new Error("Storage committed without adopting the batch"); this.fail(error); throw error; } this.#release(holds); - const publication: CommitPublication = { seq, changes }; - for (const listener of [...this.#commitListeners]) { - contained( - () => listener(publication, context), - (error) => this.report(error, LISTENER, context), - ); - } } /** After their store: forget `holds`, unpin their documents, and move the timer to the next deadline. */ @@ -954,25 +1034,36 @@ export class SessionImpl implements Session { timer.disarm = this.#timers.foreground(run, Math.max(0, at - this.#clock())); } - #publish(seq: Seq, writes: readonly StorageWrite[], documents: AdoptedDocuments, context: Context): void { + /** + * Inside `adopt`: publish a stored commit. Its telemetry events first, then the dispatch to `subscribeCommits()` + * listeners, queued before the waits and aborts the observers wake, then the Session's own observers. Never throws: + * returns the first error of an own observer, which fails the Session once `Storage.commit()` returned. + */ + #publish( + seq: Seq, + writes: readonly StorageWrite[], + documents: AdoptedDocuments, + context: Context, + ): { readonly error: unknown } | undefined { const changes: CommitChange[] | undefined = this.#commitListeners.size > 0 || this.#internalListeners.size > 0 ? [] : undefined; for (const write of writes) { switch (write.type) { case "entry": - context.telemetry.addEvent("pi.durable.entry", { - "entry.id": write.value.id, - "entry.kind": write.value.kind, - }); + this.#event("pi.durable.entry", { "entry.id": write.value.id, "entry.kind": write.value.kind }, context); changes?.push(write); break; case "submission": if (write.value.status === "done" || write.value.status === "unanswered") { - context.telemetry.addEvent("pi.durable.submission", { - "submission.id": write.value.id, - status: write.value.status, - "answer.entry_id": write.value.answer, - }); + this.#event( + "pi.durable.submission", + { + "submission.id": write.value.id, + status: write.value.status, + "answer.entry_id": write.value.answer, + }, + context, + ); } changes?.push(write); break; @@ -981,64 +1072,192 @@ export class SessionImpl implements Session { changes?.push(write); } } - if (changes === undefined) return; + if (changes === undefined) return undefined; const tables = changes.length; for (const document of documents.observed) changes.push(document); const observed: ObservedCommit = { kind: "stored", seq, changes }; - // One publication for both, unless the commit also stored held changes, which only durable listeners get. - const stored: CommitPublication = - documents.stored === documents.observed - ? observed - : { seq, changes: [...changes.slice(0, tables), ...documents.stored] }; - // The commit is durable: a failing listener neither fails the commit nor skips the others. - for (const listener of [...this.#internalListeners]) { - contained( - () => listener(observed, context), - (error) => this.fail(error, context), - ); + if (this.#commitListeners.size > 0) { + // One publication for both, unless the commit also stored held changes, which only host listeners get. + const stored: CommitPublication = + documents.stored === documents.observed + ? observed + : { seq, changes: [...changes.slice(0, tables), ...documents.stored] }; + this.#notifyHosts(stored, context); } - for (const listener of [...this.#commitListeners]) { - contained( - () => listener(stored, context), - (error) => this.report(error, LISTENER, context), - ); + // The commit is durable: a failing observer neither fails the commit nor skips the others. + let failure: { readonly error: unknown } | undefined; + for (const listener of [...this.#internalListeners]) { + // Our own observers are synchronous. + try { + listener(observed, context); + } catch (error) { + failure ??= { error }; + } + } + return failure; + } + + /** Inside `adopt`: record a commit's telemetry event. A throwing adapter is reported, as `adopt` must not throw. */ + #event(name: string, attributes: SpanAttributes, context: Context): void { + try { + context.telemetry.addEvent(name, attributes); + } catch (error) { + this.report(error, TELEMETRY, context); } } /** - * Attach a watch to one committed incarnation: check the definition, then forward this incarnation's committed - * changes, its retirement, and close. The watch's detach removes both subscriptions. + * Inside `adopt`: call the `subscribeCommits()` listeners with `publication` one microtask from now, outside the + * storage's synchronous step and, queued before the Session's own observers run, before the waits they resolve and + * the aborts they queue. + * The listeners are those registered when it runs; a throw is reported. */ - #attachDocument(definition: AnyDocToken["definition"], loaded: LoadedDocument): Watch { + #notifyHosts(publication: CommitPublication, context: Context): void { + void SETTLED.then(() => { + for (const listener of [...this.#commitListeners]) { + contained( + () => listener(publication, context), + (error) => this.report(error, LISTENER, context), + ); + } + }); + } + + /** + * On the line, as one job: load the document at `resolved` and `attach` to it, so the attachment sees every commit + * after the value it starts from; `undefined` when absent. Cancelling `context` meanwhile rejects, after `cancel` + * undid an attachment made. + */ + async #attachDocument( + definition: AnyDocToken["definition"], + resolved: { readonly id: string; readonly address: DocumentAddress }, + context: Context, + attach: (loaded: LoadedDocument) => A, + cancel: (attached: A) => void, + ): Promise { + const signal = context.abortSignal; + let cancelled = signal?.aborted ?? false; + const markCancelled = (): void => { + cancelled = true; + }; + signal?.addEventListener("abort", markCancelled, { once: true }); + try { + const attached = await this.#enqueue(async () => { + this.#assertHealthy(); + if (cancelled) throw cancellationError(signal!); + const loaded = await this.#loadDocument(definition, resolved.id, resolved.address, context); + if (cancelled) throw cancellationError(signal!); + return loaded === undefined ? undefined : attach(loaded); + }); + if (attached === undefined) return undefined; + if (cancelled) { + cancel(attached); + throw cancellationError(signal!); + } + return attached; + } finally { + signal?.removeEventListener("abort", markCancelled); + } + } + + /** + * Follow the address of the incarnation `loaded`, after checking the definition, with the observer `create` makes + * from its value, the idempotent detach, and the report of a listener error; returns that observer. It gets each + * committed change there, a retirement as a `retired` change with the last value it was given and no ops (a change + * in the retiring commit is not published), and a later incarnation as a root replacement. When the commit makes the + * owner task of a task document terminal, no later commit can change the address: after its changes the commit + * subscription and the pin go, and `ownerEnded` tells the observer, which stays subscribed to the Session's close + * until it ends (its detach drops that). `end` gets the Session's close. The current incarnation is pinned in the + * cache. The detach runs before `end`; once it ran, nothing more of the commit underway reaches the observer or pins + * anything for it. `create` and `ownerEnded` are module functions where they can be, so a watch keeps no closure of + * its caller. + */ + #observeDocument>>( + definition: AnyDocToken["definition"], + loaded: LoadedDocument, + create: (value: Readonly, detach: () => void, report: (error: unknown) => void) => O, + ownerEnded: (observer: O) => void, + ): O { checkRecordScope(definition, loaded.record); checkRecordVersion(definition, loaded.record, loaded.storedVersion); + // The closures below keep only the address, as the cache's ID and first record, which the cache shares while the + // document is cached: `loaded` holds its incarnation's tracker, which a retirement frees. + const addressId = loaded.addressId; + const address: DocumentAddress = loaded.record; + let incarnation: DocumentId | undefined = loaded.record.id; + // The value of the incarnation followed; none while the address is absent, so the Session keeps no retired value. + let value: Readonly | undefined = loaded.tracker.value; + // The definition version the observer's value has: a change of another version replaces it whole. + let version = loaded.valueVersion; + // The incarnation observed stays cached; once it retires or the observer detaches, it is idle again. + let unpin = this.#pinDocument(addressId, incarnation); let unsubscribeCommit = (): void => {}; let unsubscribeClose = (): void => {}; - // Attached, the incarnation stays cached; once its last observer detaches, or it retires, it is idle again. - const unpin = this.#pinDocument(loaded.addressId, loaded.record.id); + let attached = true; const detach = (): void => { + if (!attached) return; + attached = false; unsubscribeCommit(); unsubscribeClose(); unpin(); }; - const observer = new Watch(loaded.tracker.value, detach, (error) => this.report(error)); - const observed = { version: loaded.valueVersion }; + const observer = create(loaded.tracker.value, detach, (error) => this.report(error)); unsubscribeCommit = this.observeCommits((publication, context) => { + // Listeners run from a copy of the set: another observer's call may have detached this one. + if (!attached) return; + let ownerEnds = false; for (const change of publication.changes) { - if (change.type !== "document" || change.record.id !== loaded.record.id) continue; - if (change.value === null) { - // A retired incarnation never changes again: its watch delivers the retirement but pins nothing. - unpin(); - observer.retire(context); + if (change.type === "task") { + const scope = address.scope; + if ( + scope.kind === "task" && + change.value.id === scope.taskId && + change.value.state.status === "terminal" + ) { + ownerEnds = true; + } continue; } - const ops = observedOperations(observed, change); - // A migration-only base changes nothing for an observer of the new version. - if (ops.length === 0) continue; - observer.advance(change.value, ops, context); + // The incarnation followed is at the address by definition; only another record needs the address compared. + if (change.type !== "document") continue; + if (change.record.id !== incarnation && !atAddress(change.record, address)) continue; + if (change.value === null) { + if (change.record.id !== incarnation) continue; + // Set with every incarnation. + const last = value!; + incarnation = undefined; + value = undefined; + unpin(); + observer.advance({ value: last, ops: NO_OPERATIONS, retired: true }, context); + } else if (change.record.id !== incarnation || change.version !== version) { + // A new incarnation at the address, or a value of another definition version, whose shape the observer's + // value does not have: shown whole. + if (change.record.id !== incarnation) { + incarnation = change.record.id; + unpin(); + unpin = this.#pinDocument(addressId, incarnation); + } + version = change.version!; + value = change.value; + observer.advance({ value, ops: [["r", value]], retired: false }, context); + } else { + // A migration-only base changes nothing for an observer of the new version. + if (change.ops.length === 0) continue; + value = change.value; + observer.advance({ value, ops: change.ops, retired: false }, context); + } + // The observer may detach in its call, or throw and be detached: the rest of the commit is not its. + if (!attached) return; } + if (!ownerEnds) return; + unsubscribeCommit(); + unpin(); + ownerEnded(observer); + }); + unsubscribeClose = this.subscribeClose(() => { + detach(); + observer.end(this.#failure); }); - unsubscribeClose = this.subscribeClose(() => observer.end(this.#failure)); return observer; } @@ -1113,16 +1332,98 @@ export function contained(callback: () => unknown, onError: (error: unknown) => } /** - * Operations an observer applies for one committed change. An observer hydrated under another definition version holds - * a differently shaped value, so it receives the new value as a root replacement instead of operations for that shape. + * `observer`, a host's, behind the rule for observers that run inside Storage's `adopt`: one that throws is reported + * through `report`, detached, and ended with the error, so the throw never reaches `adopt`; nothing reaches it after + * its detach or end, and from then on the guard references neither it nor its source. `detach` is the source's; the + * returned `detach()` is the observation's. */ -function observedOperations( - observed: { version: number }, - change: Extract, -): readonly Op[] { - if (change.version === observed.version) return change.ops; - observed.version = change.version!; - return [["r", change.value]]; +export function guardObserver( + observer: Observer, + detach: () => void, + report: (error: unknown) => void, +): Observer & { detach(): void } { + type Attached = { + readonly observer: Observer; + readonly detach: () => void; + readonly report: (error: unknown) => void; + }; + let attached: Attached | undefined = { observer, detach, report }; + /** Detach from the source once, and hand back what it was attached with. */ + const release = (): Attached | undefined => { + const released = attached; + attached = undefined; + released?.detach(); + return released; + }; + return { + advance: (change, context) => { + const current = attached; + if (current === undefined) return; + try { + current.observer.advance(change, context); + } catch (error) { + current.report(error); + if (release() !== undefined) contained(() => current.observer.end({ error }), current.report); + } + }, + end: (failure) => { + const ending = release(); + if (ending !== undefined) contained(() => ending.observer.end(failure), ending.report); + }, + detach: () => { + release(); + }, + }; +} + +/** + * The `Observation` of a guarded observer attached at `value`. Built outside the attaching closure, so a detached + * observation keeps nothing but `value` and the spent guard alive. + */ +export function observation(value: T, guarded: { detach(): void }): Observation { + return { value, detach: () => guarded.detach() }; +} + +/** A document watch over its source's detach and report. */ +function createWatch( + value: Readonly, + detach: () => void, + report: (error: unknown) => void, +): Watch> { + return new Watch>(value, detach, report); +} + +/** Undo a watch attached while its acquisition was cancelled. */ +function cancelWatch(watch: Watch>): void { + watch.cancel(); +} + +/** Undo an observation attached while its attachment was cancelled. */ +function detachObservation(attached: Observation>): void { + attached.detach(); +} + +/** A task document's watch delivers the retirement that ends its task, then ends. */ +function endWatch(watch: Watch>): void { + watch.endAfterDelivery(); +} + +/** A task document's observer ends right after the retirement that ends its task. */ +function endGuarded(guarded: Observer>): void { + guarded.end(undefined); +} + +/** Whether `record` is an incarnation at `address`. */ +function atAddress(record: DocumentRecord, address: DocumentAddress): boolean { + if (record.kind !== address.kind || record.key !== address.key) return false; + switch (record.scope.kind) { + case "session": + return address.scope.kind === "session"; + case "conversation": + return address.scope.kind === "conversation" && address.scope.conversationId === record.scope.conversationId; + case "task": + return address.scope.kind === "task" && address.scope.taskId === record.scope.taskId; + } } function cancellationError(signal: AbortSignal): Error { diff --git a/packages/durable/src/testing/env-conformance.ts b/packages/durable/src/testing/env-conformance.ts index a2c7d43ba..1e6d1ad1a 100644 --- a/packages/durable/src/testing/env-conformance.ts +++ b/packages/durable/src/testing/env-conformance.ts @@ -2,12 +2,12 @@ import type { Context } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/context"; import { type ExecutionEnv, + type FileChange, type FileInfo, getOrThrow, type ListFilesOptions, type Result, type ShellOutputInfo, - type WatchChange, type WatchTarget, } from "../env/index.ts"; import type { EnvConformanceCase, EnvConformanceOptions } from "./types.ts"; @@ -63,7 +63,7 @@ async function listed( } /** Whether a change reports `path`: an overflow, or a reported path at or above it. */ -function covers(change: WatchChange, path: string): boolean { +function covers(change: FileChange, path: string): boolean { if ("overflow" in change) return true; if (!("paths" in change)) return false; return change.paths.some( @@ -76,13 +76,13 @@ async function watching( env: ExecutionEnv, targets: readonly WatchTarget[], run: (helpers: { - readonly changes: WatchChange[]; + readonly changes: FileChange[]; /** Wait up to three seconds for a change, after the call, that reports `path`. */ expectChange(path: string, change: () => Promise): Promise; absolute(path: string): Promise; }) => Promise, ): Promise { - const changes: WatchChange[] = []; + const changes: FileChange[] = []; const watcher = getOrThrow(await env.watch(targets, (change) => changes.push(change), context)); try { const absolute = async (path: string) => getOrThrow(await env.absolutePath(path, context)); @@ -384,7 +384,7 @@ export function createEnvConformance(options: EnvConformanceOptions): readonly E }), watchCase("watch stops reporting once closed", async (env) => { - const changes: WatchChange[] = []; + const changes: FileChange[] = []; const watcher = getOrThrow(await env.watch([{ path: "file.txt" }], (change) => changes.push(change), context)); assert.ok(watcher.mode === "native" || watcher.mode === "polling", "watcher reports its mode"); await watcher.close(context); diff --git a/packages/durable/src/types.ts b/packages/durable/src/types.ts index 5844fccb1..97bdcb75e 100644 --- a/packages/durable/src/types.ts +++ b/packages/durable/src/types.ts @@ -1,4 +1,4 @@ -import type { Context, Draft, JsonValue, WatchHandle } from "@earendil-works/chord"; +import type { Context, Draft, JsonValue, WatchChange, WatchHandle } from "@earendil-works/chord"; import type { Op } from "@earendil-works/chord/delta"; import type { Message, Models } from "@earendil-works/pi-ai"; import type { TelemetryCarrier } from "@earendil-works/pi-telemetry"; @@ -142,8 +142,11 @@ export type ReportTask = { readonly conversationId: ConversationId; }; -/** A host callback, by its `HarnessOptions` name; `env` is the environment builder, `models` a provider call. */ -export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake"; +/** + * A host callback, by its `HarnessOptions` name; `env` is the environment builder, `models` a provider call, and + * `telemetry` a telemetry adapter call made while a commit is published, which must not fail it. + */ +export type HostCallback = "now" | "settings" | "registry" | "env" | "models" | "setTimer" | "onWake" | "telemetry"; /** * What a report to `HarnessOptions.onReport` is about. `extension`, where present, names the extension whose code @@ -1013,10 +1016,38 @@ export interface Tx { } /** - * Watch of one committed document incarnation. A retirement delivers `[["r", null]]` with value `null`, then ends the - * watch as `retired`; the Session's close ends it as `ended`, its failure as `failed` with the error that failed it. + * Watch of one document address (kind, scope, and key), from the incarnation found at acquisition on. A retirement is a + * call with `retired` set, the last value the watch was given and no ops (a change made in the retiring commit itself + * is not shown: that commit publishes the retirement only); the watch goes on, and a document created at the address + * later arrives as `[["r", value]]`, folded with what is pending. It ends `ended` when its scope's owner ends: a task + * document's watch once the retirement call of the commit that makes its task terminal returned, every watch at Session + * close (a pending retirement dropped); `failed` with the error that failed the Session. */ -export type DocumentWatch = WatchHandle | null>; +export type DocumentWatch = WatchHandle>; + +/** + * A synchronous observer of committed Durable state, for adapters such as a Chord source (`observeDoc()`, + * `Harness.observeConversation()`, `Harness.observeTaskGraph()`). `advance` gets every change, unfolded, as memory + * adopts it: for a stored commit inside Storage's `adopt`, for a deferred one before it is stored. A view or the graph + * gets at most one call per commit, a document observer one per change at its address (two for a commit that retires + * and recreates it); the task graph also advances outside any commit and `adopt`, on the Session line, when a registry + * change or a failed migration changes a `blocked` reason. Calls may come before the attaching promise resolves; their + * ops apply from `Observation.value`. `advance` must only record the change, synchronously and quickly, never await, + * call the Session, or throw. A throw is reported as `listener_error`, detaches the observer, and ends it with the + * error. `end` is called once, when the Session closes or fails (`failure`), or when the observed scope's owner ends (a + * task document's task, after its retirement change, inside that commit's `adopt`), and never after `detach()`. + */ +export type Observer = { + advance(change: WatchChange, context: Context): void; + end(failure: { readonly error: unknown } | undefined): void; +}; + +/** An attached `Observer`: the value at attachment, which every later `advance` starts from, and how to detach it. */ +export type Observation = { + readonly value: T; + /** Idempotent: no `advance` or `end` follows it, not even for the rest of a commit underway. */ + detach(): void; +}; /** Committed document reads. */ export type DocumentReader = Pick; @@ -1088,8 +1119,55 @@ export interface Session extends DocumentObserver { */ readonly closed: Promise; /** - * Observe stored commits synchronously after adoption, each with exactly what it stored; a commit held in memory is - * published when it is stored (`CommitPublication`). The listener must not block or call Session APIs. + * Attach a synchronous `Observer` to one document address, atomically on the Session line, as `watchDoc()` acquires + * a watch: the observation's value includes every commit before it, and `advance` gets every later one, retirements + * and later incarnations included. `undefined` when the address is absent. `context` governs only the attachment: + * cancelled while attaching, the promise rejects and nothing stays attached. + */ + observeDoc( + token: SessionDocToken, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocToken, + conversationId: ConversationId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocToken, + taskId: TaskId, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: SessionDocFamilyToken, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: ConversationDocFamilyToken, + conversationId: ConversationId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + observeDoc( + token: TaskDocFamilyToken, + taskId: TaskId, + key: string, + observer: Observer>, + context: Context, + ): Promise> | undefined>; + /** + * Observe stored commits, each with exactly what it stored, except that a document the commit retires is published + * as its retirement only; a commit held in memory is published when it is stored (`CommitPublication`). Storage's + * adoption of the commit queues one dispatch, which runs a microtask later, before every wait the commit resolves + * and every abort it queues, and calls the listeners registered by then; a commit adopted while none was registered + * has no dispatch. There is no order relative to watch listeners. The listener runs synchronously and must not block + * or call Session APIs. */ subscribeCommits(listener: (publication: CommitPublication, context: Context) => void): () => void; /** @@ -1159,8 +1237,9 @@ export interface Storage { /** * Atomically persist one batch and return its sequence. Once the batch is durable, call `adopt(seq)` exactly once, * synchronously and before this storage's next commit starts, then resolve with `seq`; a commit that fails before it - * is durable never calls `adopt`. `adopt` is the caller's in-memory adoption of the batch, so memory always shows a - * complete prefix of the log; it must not throw. Once resolved, later reads through this storage observe the batch. + * is durable never calls `adopt`. `adopt` is the caller's in-memory adoption and publication of the batch, so memory + * and the Session's views always show a complete prefix of the log; it runs the Session's own observers, which call + * no Storage method, and never throws. Once resolved, later reads through this storage observe the batch. */ commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, context: Context): Promise; diff --git a/packages/durable/test/env-node-conformance.test.ts b/packages/durable/test/env-node-conformance.test.ts index 38e8de61b..5bdbe15c5 100644 --- a/packages/durable/test/env-node-conformance.test.ts +++ b/packages/durable/test/env-node-conformance.test.ts @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context"; import { afterEach, describe, expect, it } from "vitest"; -import { getOrThrow, type WatchChange } from "../src/env/index.ts"; +import { type FileChange, getOrThrow } from "../src/env/index.ts"; import { NodeExecutionEnv } from "../src/env/node.ts"; import { registerEnvConformance } from "../src/testing/index.ts"; @@ -72,7 +72,7 @@ describe("NodeExecutionEnv watch limits", () => { }); getOrThrow(await env.remove("tree/c", { recursive: true }, context)); - const changes: WatchChange[] = []; + const changes: FileChange[] = []; const watcher = getOrThrow( await env.watch([{ path: "tree", recursive: true }], (change) => changes.push(change), context), ); diff --git a/packages/durable/test/examples/04-watches.ts b/packages/durable/test/examples/04-watches.ts index bc76fc6ec..eed3a17cc 100644 --- a/packages/durable/test/examples/04-watches.ts +++ b/packages/durable/test/examples/04-watches.ts @@ -32,7 +32,7 @@ console.log("watch baseline:", notesWatch.value); const busy = Promise.withResolvers(); const done = Promise.withResolvers(); let calls = 0; -notesWatch.start(async (value, ops, _deliveryContext) => { +notesWatch.start(async ({ value, ops }, _deliveryContext) => { console.log("watch update:", value, JSON.stringify(ops)); // The first call stays busy, as a slow consumer would, while three more commits land. if (++calls === 1) await busy.promise; diff --git a/packages/durable/test/examples/15-real-model.ts b/packages/durable/test/examples/15-real-model.ts index b5cbe1244..8adb2bf50 100644 --- a/packages/durable/test/examples/15-real-model.ts +++ b/packages/durable/test/examples/15-real-model.ts @@ -45,7 +45,7 @@ if (process.env.OPENAI_API_KEY === undefined) { process.stdout.write(text.slice(printed.length)); printed = text; }; - liveWatch.start(async (value) => { + liveWatch.start(async ({ value }) => { const block = value?.generation?.message?.content.find((content) => content.type === "text"); if (block?.type === "text") printText(block.text); }); diff --git a/packages/durable/test/examples/18-json.ts b/packages/durable/test/examples/18-json.ts index f1348cfe1..4e07a1768 100644 --- a/packages/durable/test/examples/18-json.ts +++ b/packages/durable/test/examples/18-json.ts @@ -82,14 +82,14 @@ let stop: () => Promise; if (mode === "events") { const stream = await watchEvents(harness, root.id, context); print(stream.snapshot); - stream.start(async (events) => { + stream.start(async ({ value: events }) => { for (const event of events) print(event); }); stop = () => stream.stop(); } else { const watch = await root.watch(context); print({ view: watch.value }); - watch.start(async (_value, ops) => print({ ops })); + watch.start(async ({ ops }) => print({ ops })); stop = () => watch.stop(); } diff --git a/packages/durable/test/examples/20-late-join.ts b/packages/durable/test/examples/20-late-join.ts index 122ad70ee..53c7118cf 100644 --- a/packages/durable/test/examples/20-late-join.ts +++ b/packages/durable/test/examples/20-late-join.ts @@ -7,7 +7,6 @@ import { Type } from "@earendil-works/pi-ai"; import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { - type AgentEvent, createRegistry, defineExtension, defineTool, @@ -63,7 +62,7 @@ console.log( view.value.entries.map((entry) => entry.kind), ); console.log("view tool slot:", live.tools?.[0]?.status, JSON.stringify(live.tools?.[0]?.output)); -view.start(async (value) => { +view.start(async ({ value }) => { const slot = (value.docs["pi.live"] as LiveState).tools?.[0]; if (slot?.status === "running") console.log("view output now:", JSON.stringify(slot.output)); }); @@ -75,7 +74,7 @@ console.log( stream.snapshot.tools.map((slot) => `${slot.name} ${slot.status}`), ); let output = stream.snapshot.tools[0]?.output ?? ""; -stream.start(async (events: readonly AgentEvent[]) => { +stream.start(async ({ value: events }) => { for (const event of events) { if (event.type === "tool_execution_update" && event.output !== undefined) { output = diff --git a/packages/durable/test/examples/21-subagent-foreground.ts b/packages/durable/test/examples/21-subagent-foreground.ts index c766803fb..e9404bb3f 100644 --- a/packages/durable/test/examples/21-subagent-foreground.ts +++ b/packages/durable/test/examples/21-subagent-foreground.ts @@ -114,7 +114,7 @@ const attached = new Set(); const attach = async (id: ConversationId, indent: string): Promise => { attached.add(id); const stream = await watchEvents(harness, id, context); - stream.start(async (events) => { + stream.start(async ({ value: events }) => { for (const event of events) { print(indent, event); if (event.type !== "tool_execution_update") continue; diff --git a/packages/durable/test/examples/22-subagent-background.ts b/packages/durable/test/examples/22-subagent-background.ts index 19fba5dc5..0b3925d9e 100644 --- a/packages/durable/test/examples/22-subagent-background.ts +++ b/packages/durable/test/examples/22-subagent-background.ts @@ -14,7 +14,6 @@ import { createModels } from "@earendil-works/pi-ai/models"; import { fauxAssistantMessage, fauxProvider, fauxText, fauxToolCall } from "@earendil-works/pi-ai/providers/faux"; import { openaiProvider } from "@earendil-works/pi-ai/providers/openai"; import { - type AgentEvent, AssistantEntry, type ConversationId, configure, @@ -285,7 +284,7 @@ function contentText(content: string | readonly { type: string; text?: string }[ /** Print the main conversation's messages as they are committed. Subagents work in their own conversations. */ async function follow(harness: Harness, id: ConversationId): Promise { const stream = await watchEvents(harness, id, context); - stream.start(async (events: readonly AgentEvent[]) => { + stream.start(async ({ value: events }) => { for (const event of events) { if (event.type !== "message_end") continue; const message = event.entry.model?.[0]; diff --git a/packages/durable/test/harness-compaction.test.ts b/packages/durable/test/harness-compaction.test.ts index 61f1f1d14..0cfa7426b 100644 --- a/packages/durable/test/harness-compaction.test.ts +++ b/packages/durable/test/harness-compaction.test.ts @@ -1012,7 +1012,7 @@ describe("background threshold compaction", () => { chat.setup.settings.compaction = BACKGROUND; const events: AgentEvent[] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); chat.faux.summaries.push(failure("bad request")); @@ -1181,7 +1181,7 @@ describe("blocking threshold compaction", () => { chat.faux.summaries.push(gated(deferred(), summary(), reached)); const events: AgentEvent[] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); const input = await chat.root.submit({ type: "input", content: text("u4", 200) }, context); @@ -1572,7 +1572,7 @@ describe("compaction events and live status", () => { chat.setup.settings.retry = { enabled: true, maxRetries: 2, baseDelayMs: 60_000 }; const events: AgentEvent[] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); chat.faux.summaries.push(failure("overloaded")); @@ -2033,7 +2033,7 @@ describe("compaction edge cases", () => { await history(chat); const events: AgentEvent[] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); chat.faux.summaries.push(summary()); @@ -2052,7 +2052,7 @@ describe("compaction edge cases", () => { chat.setup.settings.compaction = BACKGROUND; const events: AgentEvent[][] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push([...batch]); }); // A new section makes the preparation commit append a system entry next to the compaction's status. @@ -2358,7 +2358,7 @@ describe("blocked compaction", () => { const chat = { harness, root, setup, faux: script(setup) }; const events: AgentEvent[] = []; const stream = await watchEvents(chat.harness, chat.root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); const inspection = await chat.harness.inspect(context); diff --git a/packages/durable/test/harness-events.test.ts b/packages/durable/test/harness-events.test.ts index 029a6a53d..194da5515 100644 --- a/packages/durable/test/harness-events.test.ts +++ b/packages/durable/test/harness-events.test.ts @@ -35,7 +35,7 @@ import { aborted, deferred } from "./task-support.ts"; async function listen(harness: Harness, conversation: Conversation) { const stream = await watchEvents(harness, conversation.id, context); const batches: AgentEvent[][] = []; - stream.start(async (events) => { + stream.start(async ({ value: events }) => { batches.push([...events]); }); return { stream, batches, events: () => batches.flat() }; @@ -259,7 +259,7 @@ describe("agent events", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: { index } }), context); } const batches: AgentEvent[][] = []; - stream.start(async (events) => { + stream.start(async ({ value: events }) => { batches.push([...events]); }); await drained(); @@ -278,7 +278,7 @@ describe("agent events", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); } const batches: AgentEvent[][] = []; - stream.start(async (events) => { + stream.start(async ({ value: events }) => { batches.push([...events]); }); await drained(); @@ -540,7 +540,7 @@ describe("agent events", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); } const batches: AgentEvent[][] = []; - stream.start(async (events) => { + stream.start(async ({ value: events }) => { batches.push([...events]); }); await drained(); diff --git a/packages/durable/test/harness-generation-recovery.test.ts b/packages/durable/test/harness-generation-recovery.test.ts index 468723ce4..f517bbe12 100644 --- a/packages/durable/test/harness-generation-recovery.test.ts +++ b/packages/durable/test/harness-generation-recovery.test.ts @@ -130,8 +130,8 @@ describe("generation recovery", () => { opened.harness.resume(); const watch = (await opened.harness.watchDoc(LiveDoc, opened.root.id, context))!; let watched: string | undefined; - watch.start(async (value) => { - watched = textOf(value?.generation?.message as Message) ?? watched; + watch.start(async ({ value }) => { + watched = textOf(value.generation?.message as Message) ?? watched; }); const id = (await opened.root.submit({ type: "input", content: "hi" }, context)).id; await waitFor(() => watched !== undefined); diff --git a/packages/durable/test/harness-input-entries.test.ts b/packages/durable/test/harness-input-entries.test.ts index 019172d03..7c954e6f1 100644 --- a/packages/durable/test/harness-input-entries.test.ts +++ b/packages/durable/test/harness-input-entries.test.ts @@ -265,7 +265,7 @@ describe("data-only writes", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const events: AgentEvent[] = []; const stream = await watchEvents(harness, root.id, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); const input = await root.submit({ type: "input", content: "a" }, context); diff --git a/packages/durable/test/harness-nested-tools-restart.test.ts b/packages/durable/test/harness-nested-tools-restart.test.ts index 342e103c0..68a96b854 100644 --- a/packages/durable/test/harness-nested-tools-restart.test.ts +++ b/packages/durable/test/harness-nested-tools-restart.test.ts @@ -79,7 +79,7 @@ function results(entries: Awaited>): ToolResultMes async function collectEnds(harness: Harness, conversationId: Parameters[0]) { const ends: Extract[] = []; const stream = await watchEvents(harness, conversationId, context); - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { for (const event of batch) if (event.type === "tool_execution_end") ends.push(event); }); return { ends, stop: () => stream.stop() }; diff --git a/packages/durable/test/harness-nested-tools.test.ts b/packages/durable/test/harness-nested-tools.test.ts index 5d68eb34a..d8354fcf2 100644 --- a/packages/durable/test/harness-nested-tools.test.ts +++ b/packages/durable/test/harness-nested-tools.test.ts @@ -374,7 +374,7 @@ describe("nested tool calls", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const stream = await watchEvents(harness, root.id, context); const ends: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { ends.push(...batch.filter((event) => event.type === "tool_execution_end")); }); const settled = await (await root.submit({ type: "input", content: "go" }, context)).wait(context); @@ -455,7 +455,7 @@ describe("nested tool calls", () => { const stream = await watchEvents(harness, root.id, context); expect(stream.snapshot.nestedTools).toEqual([]); const events: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); const submission = await root.submit({ type: "input", content: "go" }, context); @@ -833,7 +833,7 @@ describe("nested tool calls", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const stream = await watchEvents(harness, root.id, context); const tools: (string | undefined)[][] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { for (const event of batch) { if (event.type === "tool_execution_start" || event.type === "tool_execution_end") { tools.push([event.type, event.toolCallId, event.parentToolCallId]); @@ -895,7 +895,7 @@ describe("nested tool calls", () => { ["c1/2", "running"], ]); const updates: [string, unknown][] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { for (const event of batch) { if (event.type === "tool_execution_update" && event.output !== undefined) { updates.push([event.toolCallId, event.output]); @@ -990,7 +990,7 @@ describe("nested tool calls", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const stream = await watchEvents(harness, root.id, context); const ends: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { ends.push(...batch.filter((event) => event.type === "tool_execution_end")); }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); @@ -1040,7 +1040,7 @@ describe("nested tool calls", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const stream = await watchEvents(harness, root.id, context); const starts: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { starts.push(...batch.filter((event) => event.type === "tool_execution_start")); }); const submission = await root.submit({ type: "input", content: "go" }, context); diff --git a/packages/durable/test/harness-observers.test.ts b/packages/durable/test/harness-observers.test.ts new file mode 100644 index 000000000..c99e1aa3c --- /dev/null +++ b/packages/durable/test/harness-observers.test.ts @@ -0,0 +1,459 @@ +// Tests for the Harness's synchronous observers (`observeConversation()`, `observeTaskGraph()`, `observeDoc()` of a task +// document) and for task document watches, which end with their task. +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { WatchChange } from "@earendil-works/chord"; +import { withAbortSignal } from "@earendil-works/chord/context"; +import { + type ConversationView, + defineDoc, + defineExtension, + defineTask, + type Observer, + type TaskGraph, + type TaskId, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import type { SessionImpl } from "../src/session/session.ts"; +import { ControlledStorage, context, flush } from "./session-support.ts"; +import { completed, deferred, eventually, openTasks, settled } from "./task-support.ts"; + +type Step = { phase: "run" }; + +const Progress = defineDoc<{ lines: string[] }>({ + checkpointWhen: () => false, + kind: "test.observers.progress", + version: 1, + scope: "task", + initial: () => ({ lines: [] }), +}); + +/** A task that writes its progress document, waits for `gate`, then completes, which retires the document. */ +function writer(name: string, gate: Promise, written: () => void) { + return defineTask({ + name, + version: 1, + initial: () => ({ phase: "run" }), + phases: { + run: async (task, runtime, ctx) => { + await runtime.commit(async (tx) => { + (await tx.doc(Progress, task.id)).lines.push("started"); + }, ctx); + written(); + await gate; + await runtime.commit(() => completed(null), ctx); + }, + }, + abort: async () => {}, + }); +} + +/** An observer recording each call and whether it ran inside the storage's `adopt`. */ +function recorder(storage: ControlledStorage) { + const calls: Array<{ readonly advance: WatchChange; readonly adopting: boolean } | { readonly end: unknown }> = + []; + const observer: Observer = { + advance: (change) => { + calls.push({ advance: change, adopting: storage.adopting }); + }, + end: (failure) => { + calls.push({ end: failure }); + }, + }; + return { calls, observer }; +} + +/** A Harness with a writer task started in the root conversation and its progress document written. */ +async function openWriter() { + const storage = new ControlledStorage(); + const gate = deferred(); + const written = deferred(); + const Writer = writer("test.observers.writer", gate.promise, written.resolve); + const opened = await openTasks(storage, [Writer]); + const root = await opened.harness.root(context); + const id: TaskId = await root.commit( + (tx) => tx.createTask(Writer, null, { ownership: { kind: "conversation" } }), + context, + ); + opened.harness.resume(); + await written.promise; + return { ...opened, storage, root, id, finish: gate.resolve }; +} + +const collect = (() => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + return async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; +})(); + +describe("observeConversation and observeTaskGraph", () => { + it("start at the current value and advance inside adopt with the same change a watch gets", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const view = recorder(storage); + const viewObservation = await harness.observeConversation(root.id, view.observer, context); + const graph = recorder(storage); + const graphObservation = await harness.observeTaskGraph(graph.observer, context); + expect(viewObservation.value.conversation.id).toBe(root.id); + expect(graphObservation.value).toEqual({ tasks: {} }); + const watch = await root.watch(context); + const watched: WatchChange[] = []; + watch.start(async (change) => { + watched.push(change); + }); + const entry = await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + expect(view.calls).toHaveLength(1); + const [call] = view.calls as [{ advance: WatchChange; adopting: boolean }]; + expect(call.adopting).toBe(true); + expect(call.advance.retired).toBe(false); + expect(call.advance.value.entries.at(-1)?.id).toBe(entry.id); + await flush(); + expect(watched).toEqual([call.advance]); + // No task changed: the graph got nothing. + expect(graph.calls).toEqual([]); + viewObservation.detach(); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + expect(view.calls).toHaveLength(1); + await watch.stop(); + await harness.close(context); + expect(graph.calls).toEqual([{ end: undefined }]); + }); + + it("end with the error that failed the Harness", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const graph = recorder(storage); + await harness.observeTaskGraph(graph.observer, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "disk gone", + ); + await harness.closed; + expect(graph.calls).toEqual([{ end: { error: failure } }]); + }); + + it("report a throwing observer, detach it and end it with the error, and keep the view going", async () => { + const storage = new ControlledStorage(); + const { harness, reports, infos } = await openTasks(storage, []); + const root = await harness.root(context); + const bug = new Error("observer bug"); + const ends: unknown[] = []; + await harness.observeConversation( + root.id, + { + advance: () => { + throw bug; + }, + end: (failure) => ends.push(failure), + }, + context, + ); + const healthy = recorder(storage); + await harness.observeConversation(root.id, healthy.observer, context); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + expect(reports).toEqual([bug]); + expect(infos).toEqual([{ type: "listener_error" }]); + expect(ends).toEqual([{ error: bug }]); + expect(healthy.calls).toHaveLength(2); + await harness.close(context); + expect(ends).toHaveLength(1); + expect(await harness.closed).toEqual({ reason: "closed" }); + }); + + it("reject, never throw, on a closed Harness", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + await harness.close(context); + const view = harness.observeConversation(root.id, recorder(storage).observer, context); + const graph = harness.observeTaskGraph(recorder(storage).observer, context); + await expect(view).rejects.toThrow("Session is closed"); + await expect(graph).rejects.toThrow("Session is closed"); + }); + + it("reject an attachment cancelled while it waits on the line, and attach nothing", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + const view = recorder(storage); + const graph = recorder(storage); + // A commit holds the line, so both attachments wait behind it. + const held = storage.holdCommits(); + const committing = root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + await held.entered; + const controller = new AbortController(); + const cancelled = withAbortSignal(controller.signal, context); + const attachingView = harness.observeConversation(root.id, view.observer, cancelled); + const attachingGraph = harness.observeTaskGraph(graph.observer, cancelled); + controller.abort(new Error("gone")); + held.release(); + await committing; + await expect(attachingView).rejects.toThrow("gone"); + await expect(attachingGraph).rejects.toThrow("gone"); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + await harness.close(context); + expect(view.calls).toEqual([]); + expect(graph.calls).toEqual([]); + }); + + it("advance a graph observer with a blocked refresh, outside any commit", async () => { + const Missing = defineTask({ + name: "test.observers.missing", + version: 1, + initial: () => ({ phase: "run" }), + phases: { run: (_task, runtime, ctx) => runtime.commit(() => completed(null), ctx) }, + abort: async () => {}, + }); + const storage = new ControlledStorage(); + const { harness, registry } = await openTasks(storage, []); + const graph = recorder(storage); + const observation = await harness.observeTaskGraph(graph.observer, context); + const root = await harness.root(context); + // Scheduling is not resumed: the task stays pending, blocked while its definition is missing. + const id = await root.commit( + (tx) => tx.createTask(Missing, null, { ownership: { kind: "conversation" } }), + context, + ); + const key = String(id); + expect(graph.calls).toEqual([ + { + advance: { + value: { tasks: { [key]: expect.objectContaining({ blocked: "missing_task" }) } }, + ops: [["s", ["tasks", key], expect.objectContaining({ blocked: "missing_task" })]], + retired: false, + }, + adopting: true, + }, + ]); + registry.install(defineExtension({ name: "late", tasks: [Missing] })); + await eventually(() => graph.calls.length === 2); + // The registry change re-derives the reason on the line, with no commit and outside the storage's `adopt`. + expect(graph.calls[1]).toEqual({ + advance: { value: expect.anything(), ops: [["d", ["tasks", key, "blocked"]]], retired: false }, + adopting: false, + }); + expect((graph.calls[1] as { advance: WatchChange }).advance.value.tasks[key]?.blocked).toBeUndefined(); + harness.resume(); + await harness.waitForTask(id, context); + observation.detach(); + await harness.close(context); + }); + + it("keep no detached observer alive", async () => { + const storage = new ControlledStorage(); + const { harness } = await openTasks(storage, []); + const root = await harness.root(context); + /** Attach observers only the Harness references; `detach` says whether to detach them before returning. */ + const attach = async (detach: boolean) => { + const view: Observer = { advance: () => {}, end: () => {} }; + const graph: Observer = { advance: () => {}, end: () => {} }; + const observations = [ + await harness.observeConversation(root.id, view, context), + await harness.observeTaskGraph(graph, context), + ]; + if (detach) for (const observation of observations) observation.detach(); + return { observations, view: new WeakRef(view), graph: new WeakRef(graph) }; + }; + // Control: attached observers stay, so the collection below is real. + const attached = await attach(false); + await collect(); + expect(attached.view.deref()).toBeDefined(); + expect(attached.graph.deref()).toBeDefined(); + for (const observation of attached.observations) observation.detach(); + + const detached = await attach(true); + await collect(); + expect(detached.view.deref()).toBeUndefined(); + expect(detached.graph.deref()).toBeUndefined(); + await harness.close(context); + }); +}); + +describe("task documents", () => { + it("a watch delivers the retirement of the commit that ends its task, then ends `ended`", async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + expect(watch.value).toEqual({ lines: ["started"] }); + const changes: WatchChange>[] = []; + watch.start(async (change) => { + changes.push(change); + }); + finish(); + await harness.waitForTask(id, context); + expect(await watch.closed).toEqual({ reason: "ended" }); + expect(changes).toEqual([{ value: { lines: ["started"] }, ops: [], retired: true }]); + expect(changes[0]!.value).toBe(watch.value); + // The task is terminal: its address is absent for good. + expect(await harness.watchDoc(Progress, id, context)).toBeUndefined(); + await harness.close(context); + }); + + it("an observer gets the retirement, then its end, inside the adopt of the commit that ends the task", async () => { + const { harness, storage, id, finish } = await openWriter(); + const progress = recorder>(storage); + let endedAdopting: boolean | undefined; + const observer: Observer> = { + advance: progress.observer.advance, + end: (failure) => { + endedAdopting = storage.adopting; + progress.observer.end(failure); + }, + }; + await harness.observeDoc(Progress, id, observer, context); + finish(); + await harness.waitForTask(id, context); + expect(progress.calls).toEqual([ + { advance: { value: { lines: ["started"] }, ops: [], retired: true }, adopting: true }, + { end: undefined }, + ]); + expect(endedAdopting).toBe(true); + await harness.close(context); + expect(progress.calls).toHaveLength(2); + }); + + it("keeps no task document watch alive once its task ended", async () => { + const { harness, id, finish } = await openWriter(); + /** A started watch of the task's document, which only the Harness references. */ + const acquire = async (): Promise> => { + const watch = (await harness.watchDoc(Progress, id, context))!; + watch.start(async () => {}); + return new WeakRef(watch); + }; + const watch = await acquire(); + // Control: the watch stays while its task runs, so the collection below is real. + await collect(); + expect(watch.deref()).toBeDefined(); + finish(); + await harness.waitForTask(id, context); + await collect(); + expect(watch.deref()).toBeUndefined(); + // The Harness is the Session kernel; nothing stays pinned for the ended watch. + expect((harness as unknown as SessionImpl).documentCacheSizes.attached).toBe(0); + await harness.close(context); + }); + + it("a watch never started when its task ends still ends with the Harness's close, keeping nothing of it", async () => { + /** Acquire a task document's watch, never start it, end its task, and close the Harness. */ + const acquire = async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + finish(); + await harness.waitForTask(id, context); + // Its retirement is pending until a start that never comes: the watch is open. + await flush(); + expect(await settled(watch.closed)).toBe(false); + await harness.close(context); + return { watch, harness: new WeakRef(harness) }; + }; + const { watch, harness } = await acquire(); + // Closed at the Harness's close, the pending retirement dropped, as for every watch. + expect(await watch.closed).toEqual({ reason: "ended" }); + expect(() => watch.start(async () => {})).toThrow("Watch is stopped"); + await collect(); + expect(harness.deref()).toBeUndefined(); + }); + + it("a watch never started when its task ends fails with the Harness", async () => { + const { harness, storage, root, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + finish(); + await harness.waitForTask(id, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "disk gone", + ); + expect(await watch.closed).toEqual({ reason: "failed", error: failure }); + }); + + it("a watch busy in an earlier call when its task ends fails with the Harness before its retirement call", async () => { + const { harness, storage, root, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + const entered = deferred(); + const release = deferred(); + const values: unknown[] = []; + watch.start(async ({ value }) => { + values.push(value); + entered.resolve(); + await release.promise; + }); + await harness.commit(async (tx) => { + (await tx.doc(Progress, id)).lines.push("more"); + }, context); + await entered.promise; + // The task ends while the listener is busy: its retirement waits behind the running call. + finish(); + await harness.waitForTask(id, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "disk gone", + ); + expect(await watch.closed).toEqual({ reason: "failed", error: failure }); + release.resolve(); + await flush(); + expect(values).toEqual([{ lines: ["started", "more"] }]); + }); + + it("a watch whose retirement call runs when its task ends ends only after that call returns", async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + const entered = deferred(); + const release = deferred(); + let returned = false; + watch.start(async () => { + entered.resolve(); + await release.promise; + returned = true; + }); + // An explicit retirement first: its call runs while the task ends, and nothing more is pending. + await harness.commit((tx) => tx.retireDoc(Progress, id), context); + await entered.promise; + finish(); + await harness.waitForTask(id, context); + await flush(); + expect(await settled(watch.closed)).toBe(false); + release.resolve(); + expect(await watch.closed).toEqual({ reason: "ended" }); + expect(returned).toBe(true); + await harness.close(context); + }); + + it("hands listeners changes that keep nothing of the Harness", async () => { + /** Watch a task document until its task ends, keeping every change, then close the Harness. */ + const acquire = async () => { + const { harness, id, finish } = await openWriter(); + const watch = (await harness.watchDoc(Progress, id, context))!; + const changes: WatchChange>[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await harness.commit(async (tx) => { + (await tx.doc(Progress, id)).lines.push("more"); + }, context); + await flush(); + finish(); + await harness.waitForTask(id, context); + expect(await watch.closed).toEqual({ reason: "ended" }); + await harness.close(context); + return { changes, harness: new WeakRef(harness) }; + }; + const { changes, harness } = await acquire(); + expect(changes).toHaveLength(2); + for (const change of changes) { + expect(Object.keys(change)).toEqual(["value", "ops", "retired"]); + expect(structuredClone(change)).toEqual(change); + } + await collect(); + expect(harness.deref()).toBeUndefined(); + }); +}); diff --git a/packages/durable/test/harness-storage-failure.test.ts b/packages/durable/test/harness-storage-failure.test.ts index 9169e62c7..35604a9ee 100644 --- a/packages/durable/test/harness-storage-failure.test.ts +++ b/packages/durable/test/harness-storage-failure.test.ts @@ -497,7 +497,7 @@ describe("listener and report errors", () => { ); const watch = await root.watch(context); const seen: unknown[] = []; - watch.start(async (value) => { + watch.start(async ({ value }) => { seen.push(value); }); await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); diff --git a/packages/durable/test/harness-structured-output.test.ts b/packages/durable/test/harness-structured-output.test.ts index 92ef991a5..2c3d2ba5d 100644 --- a/packages/durable/test/harness-structured-output.test.ts +++ b/packages/durable/test/harness-structured-output.test.ts @@ -165,7 +165,7 @@ describe("structured output", () => { }); const stream = await watchEvents(harness, root.id, context); const ends: NestedToolExecutionResult[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { for (const event of batch) { if (event.type === "tool_execution_end" && event.parentToolCallId !== undefined) { ends.push(event.result as NestedToolExecutionResult); diff --git a/packages/durable/test/harness-structured.test.ts b/packages/durable/test/harness-structured.test.ts index 77fa781cc..12860f89b 100644 --- a/packages/durable/test/harness-structured.test.ts +++ b/packages/durable/test/harness-structured.test.ts @@ -1310,7 +1310,7 @@ describe("tool rounds", () => { opened = harness; const stream = await watchEvents(harness, root.id, context); const events: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); @@ -1761,7 +1761,7 @@ describe("tool rounds and events", () => { async function listen(harness: Harness, conversation: Conversation) { const stream = await watchEvents(harness, conversation.id, context); const events: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); return { stream, events }; diff --git a/packages/durable/test/harness-submissions.test.ts b/packages/durable/test/harness-submissions.test.ts index 07afaa547..43831b59c 100644 --- a/packages/durable/test/harness-submissions.test.ts +++ b/packages/durable/test/harness-submissions.test.ts @@ -18,7 +18,7 @@ import { afterEach, describe, expect, it } from "vitest"; import type { SessionImpl } from "../src/session/session.ts"; import { openNodeSqliteStorage } from "../src/storage/sqlite/node.ts"; import { allEntries, chatSetup, openChat, unanswered } from "./chat-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; +import { ControlledStorage, context, LateResolvingStorage } from "./session-support.ts"; import { deferred } from "./task-support.ts"; const directories = new Set(); @@ -275,4 +275,20 @@ describe("submissions", () => { expect([UserEntry.kind, AssistantEntry.kind]).toEqual(["pi.user", "pi.assistant"]); await harness.close(context); }); + + it("resumes a wait only once subscribeCommits() listeners saw the commit that settled it", async () => { + const setup = chatSetup(); + setup.faux.setResponses([fauxAssistantMessage("hi")]); + const { harness, root } = await openChat(new LateResolvingStorage(), setup); + const settled = new Set(); + harness.subscribeCommits((publication) => { + for (const change of publication.changes) { + if (change.type === "submission" && change.value.status === "done") settled.add(change.value.id); + } + }); + const submission = await root.submit({ type: "input", content: "hello" }, context); + await submission.wait(context); + expect(settled.has(submission.id)).toBe(true); + await harness.close(context); + }); }); diff --git a/packages/durable/test/harness-task-graph.test.ts b/packages/durable/test/harness-task-graph.test.ts index 85dd66cd4..375b0ec66 100644 --- a/packages/durable/test/harness-task-graph.test.ts +++ b/packages/durable/test/harness-task-graph.test.ts @@ -5,9 +5,11 @@ import type { JsonValue } from "@earendil-works/chord"; import { applyImmutable, type Op } from "@earendil-works/chord/delta"; import { type ConversationId, + type ConversationView, defineTask, type EntryId, MemoryStorage, + type Observer, type TaskGraph, type TaskId, } from "@earendil-works/pi-durable"; @@ -90,7 +92,7 @@ describe("task graph view", () => { const seen: TaskGraph[] = []; const observe = async () => { const opened = await harness.watchTaskGraph(context); - opened.start(async (value) => { + opened.start(async ({ value }) => { seen.push(value); }); return opened; @@ -215,7 +217,7 @@ describe("task graph view", () => { // Exact frames: replaying their operations from the acquisition revision gives each delivered value. let replica: JsonValue = watch.value as unknown as JsonValue; const frames: (readonly Op[])[] = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { replica = applyImmutable(replica, ops); expect(replica).toEqual(value); frames.push(ops); @@ -334,7 +336,7 @@ describe("task graph view", () => { const second = await harness.watchTaskGraph(context); expect(second.value).toBe(first.value); const updates: string[] = []; - first.start(async (value) => { + first.start(async ({ value }) => { updates.push(value.tasks[String(id)]?.state.status ?? "gone"); }); harness.resume(); @@ -356,4 +358,29 @@ describe("task graph view", () => { await rebuilt.stop(); await harness.close(context); }); + + it("advances with the conversation view inside the one storage commit that changes both", async () => { + const storage = new ControlledStorage(); + const { Parent, Child } = family(gatesNever()); + const { harness } = await openTasks(storage, [Parent, Child]); + const root = await harness.root(context); + const seen: string[] = []; + const record = (name: string): Observer => ({ + advance: () => { + seen.push(`${name} adopting=${storage.adopting}`); + }, + end: () => {}, + }); + const view = await harness.observeConversation(root.id, record("view"), context); + const graph = await harness.observeTaskGraph(record("graph"), context); + await root.commit(async (tx) => { + await tx.appendEntry(root.id, { kind: "note" }); + await tx.createTask(Parent, null, { ownership: { kind: "conversation" } }); + }, context); + // Both inside the one adopt, in no promised order. + expect([...seen].sort()).toEqual(["graph adopting=true", "view adopting=true"]); + view.detach(); + graph.detach(); + await harness.close(context); + }); }); diff --git a/packages/durable/test/harness-tasks.test.ts b/packages/durable/test/harness-tasks.test.ts index 23c9acec8..96cf23911 100644 --- a/packages/durable/test/harness-tasks.test.ts +++ b/packages/durable/test/harness-tasks.test.ts @@ -2,6 +2,7 @@ import type { Context } from "@earendil-works/chord"; import { withAbortSignal, withCancel } from "@earendil-works/chord/context"; import { createModels, Type } from "@earendil-works/pi-ai"; import { + type CommitPublication, type Conversation, createRegistry, defineDoc, @@ -13,12 +14,14 @@ import { MemoryStorage, type RegistryReader, type RegistrySnapshot, + type StorageWrite, type TaskId, type TaskRuntime, } from "@earendil-works/pi-durable"; +import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; import { describe, expect, it } from "vitest"; import { addHooks, addTask, addTool, user } from "./harness-support.ts"; -import { ControlledStorage, context, flush } from "./session-support.ts"; +import { ControlledStorage, context, flush, LateResolvingStorage, observeCommits } from "./session-support.ts"; import { aborted, abortedWith, completed, deferred, eventually, openTasks, settled } from "./task-support.ts"; type Step = { phase: "run" }; @@ -545,8 +548,8 @@ describe("task runtime", () => { handlerContext = ctx; absent = await runtime.watchDoc(Absent, ctx); const watch = await runtime.watchDoc(Notes, ctx); - watch!.start(async (value) => { - delivered.push(value?.text ?? "retired"); + watch!.start(async ({ value }) => { + delivered.push(value.text); }); watchClosed = watch?.closed; await runtime.commit(() => completed(null), ctx); @@ -1364,3 +1367,270 @@ describe("task close", () => { expect(storage.commits.at(-1)).toMatchObject([{ type: "task", value: { id, state: { status: "pending" } } }]); }); }); + +describe("publication at adoption", () => { + /** Both storages: one that resolves right after adopting, one that still works after it (JSONL's sidecar reclaim). */ + const storages = [ + { name: "memory", open: () => new ControlledStorage() }, + { name: "work after adopt", open: () => new LateResolvingStorage() }, + ] as const; + + /** Whether `publication` makes task `id` terminal. */ + const terminal = (publication: CommitPublication, id: TaskId): boolean => + publication.changes.some( + (change) => change.type === "task" && change.value.id === id && change.value.state.status === "terminal", + ); + + /** Whether `writes` make a task terminal. */ + const endsTask = (writes: readonly StorageWrite[]): boolean => + writes.some((write) => write.type === "task" && write.value.state.status === "terminal"); + + it("resumes task and idle waits only once subscribeCommits() listeners saw the terminal commit", async () => { + const gate = deferred(); + const Gated = gated("test.wake-gated", gate.promise); + const { harness, root } = await openRoot([Gated], { storage: new LateResolvingStorage() }); + const seen = new Set(); + harness.subscribeCommits((publication) => { + for (const change of publication.changes) { + if (change.type === "task" && change.value.state.status === "terminal") seen.add(change.value.id); + } + }); + const id = await start(root, Gated); + harness.resume(); + const task = harness.waitForTask(id, context).then(() => seen.has(id)); + const idle = harness.waitForIdle(context).then(() => seen.has(id)); + const conversationIdle = root.waitForIdle(context).then(() => seen.has(id)); + gate.resolve(); + expect(await Promise.all([task, idle, conversationIdle])).toEqual([true, true, true]); + await harness.close(context); + }); + + for (const { name, open } of storages) { + it(`runs host listeners before the waits a terminal commit resolves (${name})`, async () => { + const storage = open(); + const gate = deferred(); + const Gated = gated("test.order-gated", gate.promise); + const { harness, root } = await openRoot([Gated], { storage }); + const order: string[] = []; + const id = await start(root, Gated); + harness.subscribeCommits((publication) => { + if (terminal(publication, id)) order.push(`host adopting=${storage.adopting}`); + }); + harness.resume(); + const waited = harness.waitForTask(id, context).then(() => order.push("wait")); + gate.resolve(); + await waited; + expect(order).toEqual(["host adopting=false", "wait"]); + await harness.close(context); + }); + + it(`signals a newly marked run after host listeners saw the mark, outside the storage's adopt (${name})`, async () => { + const storage = open(); + const reached = deferred(); + const order: string[] = []; + const Signalled = oneStep("test.order-signalled", async (_task, runtime) => { + runtime.signal.addEventListener("abort", () => order.push(`abort adopting=${storage.adopting}`)); + reached.resolve(); + await aborted(runtime.signal); + }); + const { harness, root } = await openRoot([Signalled], { storage }); + const id = await start(root, Signalled); + harness.subscribeCommits((publication) => { + for (const change of publication.changes) + if (change.type === "task" && change.value.id === id && change.value.abortRequested) order.push("host"); + }); + harness.resume(); + await reached.promise; + expect(await harness.abortTask(id, context)).toBe("marked"); + expect(order).toEqual(["host", "abort adopting=false"]); + await harness.close(context); + }); + } + + it("resolves a waitForIdle() called after adoption, before the storage resolved, after host listeners", async () => { + const storage = new LateResolvingStorage(); + const gate = deferred(); + const Gated = gated("test.gap-idle", gate.promise); + const { harness, root } = await openRoot([Gated], { storage }); + const order: string[] = []; + const id = await start(root, Gated); + harness.subscribeCommits((publication) => { + if (terminal(publication, id)) order.push("host"); + }); + const gapped = deferred(); + let idle: Promise | undefined; + storage.gap = (writes) => { + if (!endsTask(writes)) return false; + idle = harness.waitForIdle(context).then(() => order.push(`idle committing=${storage.committing}`)); + gapped.resolve(); + return true; + }; + harness.resume(); + gate.resolve(); + await gapped.promise; + await idle; + expect(order).toEqual(["host", "idle committing=true"]); + await harness.close(context); + }); + + it("publishes a commit the storage rejects after adopting it, resolves its waits, then fails the Session", async () => { + const storage = new LateResolvingStorage(); + const telemetry = new InMemoryTelemetryContext(); + const gate = deferred(); + const Gated = gated("test.reject-after-adopt", gate.promise); + const { harness, root, infos } = await openRoot([Gated], { storage, telemetry }); + const id = await start(root, Gated); + const seen: string[] = []; + harness.subscribeCommits((publication) => { + if (terminal(publication, id)) seen.push("terminal"); + }); + storage.rejectAfterAdopt = endsTask; + harness.resume(); + const waited = harness.waitForTask(id, context); + gate.resolve(); + expect((await waited).state.outcome).toEqual({ status: "completed", result: null }); + expect(seen).toEqual(["terminal"]); + expect(await harness.closed).toEqual({ reason: "failed", error: new Error("reclaim failed") }); + expect(infos).toContainEqual({ type: "session_failed" }); + const active = telemetry.getMetrics().filter((metric) => metric.name === "pi.durable.tasks.active"); + expect(active.map((metric) => metric.value)).toEqual([1, -1]); + }); + + it("counts a task created by a commit the storage rejects after adopting it, and its close balances it", async () => { + const storage = new LateResolvingStorage(); + const telemetry = new InMemoryTelemetryContext(); + const Idle = oneStep("test.reject-created", async () => {}); + const { harness, root } = await openRoot([Idle], { storage, telemetry }); + storage.rejectAfterAdopt = (writes) => writes.some((write) => write.type === "task"); + await expect(start(root, Idle)).rejects.toThrow("reclaim failed"); + expect(await harness.closed).toMatchObject({ reason: "failed" }); + const active = telemetry.getMetrics().filter((metric) => metric.name === "pi.durable.tasks.active"); + expect(active.map((metric) => metric.value)).toEqual([1, -1]); + }); + + it("keeps a task wait resolved when a host listener closes the Harness on the terminal commit", async () => { + const gate = deferred(); + const Gated = gated("test.close-in-listener", gate.promise); + const { harness, root } = await openRoot([Gated]); + const id = await start(root, Gated); + let closing: Promise | undefined; + harness.subscribeCommits((publication) => { + if (terminal(publication, id)) closing = harness.close(context); + }); + harness.resume(); + const waited = harness.waitForTask(id, context); + gate.resolve(); + expect((await waited).state.outcome).toEqual({ status: "completed", result: null }); + await closing; + }); + + it("keeps a task wait resolved when the Harness closes after adoption, before the storage resolved", async () => { + const storage = new LateResolvingStorage(); + const gate = deferred(); + const Gated = gated("test.close-in-gap", gate.promise); + const { harness, root } = await openRoot([Gated], { storage }); + const id = await start(root, Gated); + const gapped = deferred(); + let closing: Promise | undefined; + storage.gap = (writes) => { + if (!endsTask(writes)) return false; + closing = harness.close(context); + gapped.resolve(); + return true; + }; + harness.resume(); + const waited = harness.waitForTask(id, context).then( + (record) => record.state.outcome, + (error: unknown) => error, + ); + gate.resolve(); + await gapped.promise; + await closing; + expect(await waited).toEqual({ status: "completed", result: null }); + expect(await harness.closed).toEqual({ reason: "closed" }); + }); + + it("fails the Session with the storage error when it rejects after adopting, and reports an observer's throw", async () => { + const storage = new LateResolvingStorage(); + const { harness, root, reports, infos } = await openRoot([], { storage }); + const bug = new Error("observer bug"); + observeCommits(harness, (publication) => { + if (publication.kind === "stored" && publication.changes.some((change) => change.type === "entry")) throw bug; + }); + storage.rejectAfterAdopt = (writes) => writes.some((write) => write.type === "entry"); + await expect(root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context)).rejects.toThrow( + "reclaim failed", + ); + expect(await harness.closed).toEqual({ reason: "failed", error: new Error("reclaim failed") }); + expect(reports).toContain(bug); + expect(infos[reports.indexOf(bug)]).toEqual({ type: "listener_error" }); + expect(infos).toContainEqual({ type: "session_failed" }); + }); + + it("stores a host listener's own commit before the reconcile commit that the same publication asks for", async () => { + const storage = new ControlledStorage(); + const gate = deferred(); + const Child = gated("test.order-child", gate.promise); + const Parent = oneStep("test.order-parent", async (task, runtime, ctx) => { + await runtime.commit(async (tx) => { + await tx.createTask(Child, null, { ownership: { kind: "task", taskId: task.id } }); + }, ctx); + // The child is still live, so the outcome is held as `completing` until a reconcile pass finalizes it. + await runtime.commit(() => completed(null), ctx); + }); + const { harness, root } = await openRoot([Parent, Child], { storage }); + const parent = await start(root, Parent); + harness.resume(); + await eventually(async () => (await harness.getTask(parent, context))?.state.status === "completing"); + harness.subscribeCommits((publication) => { + const child = publication.changes.find( + (change) => + change.type === "task" && change.value.id !== parent && change.value.state.status === "terminal", + ); + if (child !== undefined) + void root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + }); + const before = storage.commits.length; + gate.resolve(); + await harness.waitForTask(parent, context); + const kinds = storage.commits.slice(before).map((writes) => { + if (writes.some((write) => write.type === "entry")) return "host entry"; + const task = writes.find((write) => write.type === "task"); + if (task?.type !== "task" || task.value.state.status !== "terminal") return "other"; + return task.value.id === parent ? "parent terminal" : "child terminal"; + }); + expect(kinds.filter((kind) => kind !== "other")).toEqual(["child terminal", "host entry", "parent terminal"]); + await harness.close(context); + }); + + it("calls a host listener added on a Harness between adopt and the dispatch only when one was registered at adopt", async () => { + const storage = new ControlledStorage(); + const { harness, root } = await openRoot([], { storage }); + const note = () => root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + const late: number[] = []; + let stopLate = (): void => {}; + // No host listener at adopt, although the Harness's own observers run: no dispatch, so the late one misses it. + storage.afterAdopt = () => { + stopLate = harness.subscribeCommits((publication) => { + late.push(publication.seq); + }); + }; + await note(); + await flush(); + expect(late).toEqual([]); + await note(); + expect(late).toHaveLength(1); + stopLate(); + // One registered at adopt: the dispatch reads the listeners when it runs, the late one included. + harness.subscribeCommits(() => {}); + const second: number[] = []; + storage.afterAdopt = () => { + harness.subscribeCommits((publication) => { + second.push(publication.seq); + }); + }; + await note(); + expect(second).toHaveLength(1); + await harness.close(context); + }); +}); diff --git a/packages/durable/test/harness-telemetry.test.ts b/packages/durable/test/harness-telemetry.test.ts index 20bf7f15f..c802568fa 100644 --- a/packages/durable/test/harness-telemetry.test.ts +++ b/packages/durable/test/harness-telemetry.test.ts @@ -11,13 +11,18 @@ import { type Storage, type TaskId, } from "@earendil-works/pi-durable"; -import { InMemoryTelemetryContext } from "@earendil-works/pi-telemetry"; +import { + InMemoryTelemetryContext, + NOOP_TELEMETRY_CONTEXT, + NOOP_TELEMETRY_METRICS, + type TelemetryContext, +} from "@earendil-works/pi-telemetry"; import { describe, expect, it, vi } from "vitest"; import { createCodemodeTool } from "../src/codemode/codemode.ts"; import { type ChatSetup, chatSetup, waitFor } from "./chat-support.ts"; import { addHooks, addTask, addTool } from "./harness-support.ts"; -import { ControlledStorage, context } from "./session-support.ts"; -import { aborted } from "./task-support.ts"; +import { ControlledStorage, context, flush } from "./session-support.ts"; +import { aborted, openTasks } from "./task-support.ts"; async function openTracked(storage: Storage, setup: ChatSetup, telemetry: InMemoryTelemetryContext) { const harness = await Harness.open( @@ -985,4 +990,85 @@ describe("Durable telemetry", () => { await harness.close(context); await sandbox.close(); }); + + it("reports telemetry adapters that throw while a commit is published, and the commit stands", async () => { + const Idle = defineTask({ + name: "test.telemetry-idle", + version: 1, + initial: () => ({ phase: "run" }), + phases: { run: async () => {} }, + abort: async () => {}, + }); + const throwingMetric = { + add: () => { + throw new Error("metrics down"); + }, + }; + const telemetry: TelemetryContext = { + ...NOOP_TELEMETRY_CONTEXT, + metrics: { ...NOOP_TELEMETRY_METRICS, upDownCounter: () => throwingMetric }, + }; + const { harness, infos } = await openTasks(new MemoryStorage(), [Idle], { telemetry }); + const root = await harness.root(context); + const view = await root.watch(context); + const shown: (number | undefined)[] = []; + view.start(async ({ value }) => { + shown.push(value.entries.at(-1)?.id); + }); + const hosts: number[] = []; + harness.subscribeCommits((publication) => { + hosts.push(publication.seq); + }); + const failing = withTelemetry( + { + ...NOOP_TELEMETRY_CONTEXT, + addEvent: () => { + throw new Error("telemetry down"); + }, + }, + context, + ); + const { entry, task } = await root.commit(async (tx) => { + const entry = await tx.appendEntry(root.id, { kind: "note", data: {} }); + const task = await tx.createTask(Idle, null, { ownership: { kind: "conversation" } }); + return { entry: entry.id, task }; + }, failing); + // The entry event, then the active-task metric; the report's own event throws too and is dropped. + expect(infos).toEqual([ + { type: "host_error", callback: "telemetry" }, + { type: "host_error", callback: "telemetry" }, + ]); + expect(hosts).toHaveLength(1); + await flush(); + expect(shown).toEqual([entry]); + expect((await harness.getTask(task, context))?.state.status).toBe("pending"); + await view.stop(); + await harness.close(context); + expect(await harness.closed).toEqual({ reason: "closed" }); + }); + + it("drops a report's own metric that throws, after onReport got the report", async () => { + const throwingCounter = { + add: () => { + throw new Error("metrics down"); + }, + }; + const telemetry: TelemetryContext = { + ...NOOP_TELEMETRY_CONTEXT, + metrics: { ...NOOP_TELEMETRY_METRICS, counter: () => throwingCounter }, + }; + const { harness, reports, infos } = await openTasks(new MemoryStorage(), [], { telemetry }); + const root = await harness.root(context); + const bug = new Error("listener bug"); + harness.subscribeCommits(() => { + throw bug; + }); + await root.commit((tx) => tx.appendEntry(root.id, { kind: "note", data: {} }), context); + // The listener runs in the commit's dispatch microtask; a throw escaping its report would be unhandled there. + await flush(); + expect(reports).toEqual([bug]); + expect(infos).toEqual([{ type: "listener_error" }]); + await harness.close(context); + expect(await harness.closed).toEqual({ reason: "closed" }); + }); }); diff --git a/packages/durable/test/harness-view.test.ts b/packages/durable/test/harness-view.test.ts index a2214c762..f07ce2bb9 100644 --- a/packages/durable/test/harness-view.test.ts +++ b/packages/durable/test/harness-view.test.ts @@ -26,7 +26,7 @@ async function record(conversation: Conversation) { const watch = await conversation.watch(context); const initial = watch.value; const frames: Frame[] = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { frames.push({ value, ops }); }); return { initial, frames, stop: () => watch.stop() }; @@ -236,7 +236,7 @@ describe("conversation view", () => { await root.commit((tx) => tx.appendEntry(root.id, { kind: "note" }), context); } const frames: Frame[] = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { frames.push({ value, ops }); }); await drained(); diff --git a/packages/durable/test/session-adoption.test.ts b/packages/durable/test/session-adoption.test.ts index 205c30066..f19ae976f 100644 --- a/packages/durable/test/session-adoption.test.ts +++ b/packages/durable/test/session-adoption.test.ts @@ -1,9 +1,21 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { basename, join } from "node:path"; import type { Context } from "@earendil-works/chord"; -import { defineDoc, MemoryStorage, type Seq, type StorageWrite } from "@earendil-works/pi-durable"; +import { + type CommitPublication, + defineDoc, + MemoryStorage, + type Seq, + type StorageWrite, +} from "@earendil-works/pi-durable"; import { afterEach, describe, expect, it, vi } from "vitest"; -import { SessionImpl } from "../src/session/session.ts"; +import type { FileError, Result } from "../src/env/index.ts"; +import { NodeExecutionEnv } from "../src/env/node.ts"; +import { type ObservedCommit, SessionImpl } from "../src/session/session.ts"; import { Transaction } from "../src/session/transaction.ts"; -import { context } from "./session-support.ts"; +import { JsonlStorage } from "../src/storage/jsonl/index.ts"; +import { ControlledStorage, context, createConversation, documentChanges } from "./session-support.ts"; const CounterDoc = defineDoc<{ count: number }>({ checkpointWhen: () => false, @@ -13,8 +25,19 @@ const CounterDoc = defineDoc<{ count: number }>({ initial: () => ({ count: 0 }), }); -/** Memory storage whose next commit, once durable, misbehaves as `fault` says. */ -class FaultyStorage extends MemoryStorage { +/** A latest, initial-fork conversation document, the kind a deferred commit may hold. */ +const NoteDoc = defineDoc<{ a?: number; b?: number }>({ + checkpointWhen: () => false, + kind: "adoption.note", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({}), +}); + +/** Controlled storage that records `adopt` and its own resolution in `order`; its next commit misbehaves as `fault` says. */ +class FaultyStorage extends ControlledStorage { fault: "reject_after_adopt" | "skip_adopt" | undefined; readonly order: string[] = []; @@ -104,3 +127,238 @@ describe("Session adoption inside Storage.commit()", () => { expect(await session.closed).toMatchObject({ reason: "failed" }); }); }); + +describe("publication at adoption", () => { + it("runs its own observers inside adopt and host listeners one microtask later, outside it, before Storage resolves", async () => { + const { storage, session } = open(); + session.observeCommits(() => { + storage.order.push(`observed adopting=${storage.adopting}`); + }); + session.subscribeCommits(() => { + storage.order.push(`host adopting=${storage.adopting}`); + }); + await bump(session); + expect(storage.order).toEqual(["adopt", "observed adopting=true", "adopted", "host adopting=false", "resolve"]); + await session.close(context); + }); + + it("calls the host listeners registered when its dispatch runs, and none for a commit adopted without one", async () => { + const { storage, session } = open(); + const calls: string[] = []; + const removed = session.subscribeCommits(() => { + calls.push("removed"); + }); + session.subscribeCommits(() => { + calls.push("kept"); + }); + storage.afterAdopt = () => { + removed(); + session.subscribeCommits(() => { + calls.push("added"); + }); + }; + await bump(session); + expect(calls).toEqual(["kept", "added"]); + // A Session with no host listener at adoption queues no dispatch, so one added before Storage resolves misses it. + const bare = open(); + const late: Seq[] = []; + bare.storage.afterAdopt = () => { + bare.session.subscribeCommits((publication) => { + late.push(publication.seq); + }); + }; + await bump(bare.session); + expect(late).toEqual([]); + await bump(bare.session); + expect(late).toHaveLength(1); + await session.close(context); + await bare.session.close(context); + }); + + it("fails the Session after Storage returned for a throw of its own observers; the others and hosts got the commit", async () => { + const { storage, session } = open(); + const bug = new Error("observer bug"); + session.observeCommits(() => { + storage.order.push("observed"); + throw bug; + }); + session.observeCommits(() => { + storage.order.push("observed after"); + }); + session.subscribeCommits(() => { + storage.order.push(`host failed=${session.failure !== undefined}`); + }); + session.subscribeClose(() => { + storage.order.push("close"); + }); + await expect(bump(session)).resolves.toBeUndefined(); + // `adopted` and `resolve`: adopt returned to Storage without throwing, and the Session failed only after that. + expect(storage.order).toEqual([ + "adopt", + "observed", + "observed after", + "adopted", + "host failed=false", + "resolve", + "close", + ]); + expect(await session.closed).toEqual({ reason: "failed", error: bug }); + const reopened = new SessionImpl(storage.reopen()); + expect(await reopened.snapshot(CounterDoc, context)).toEqual({ count: 1 }); + await reopened.close(context); + }); + + it("publishes a commit that Storage rejects after adopting it, then fails the Session", async () => { + const { storage, session } = open(); + await bump(session); + const stored: Seq[] = []; + session.observeCommits(() => { + storage.order.push("observed"); + }); + session.subscribeCommits((publication) => { + storage.order.push("host"); + stored.push(publication.seq); + }); + storage.order.length = 0; + storage.fault = "reject_after_adopt"; + await expect(bump(session)).rejects.toThrow("reclaim failed"); + expect(storage.order).toEqual(["adopt", "observed", "adopted", "host", "resolve"]); + expect(stored).toHaveLength(1); + expect(await session.closed).toMatchObject({ reason: "failed" }); + const reopened = new SessionImpl(storage.reopen()); + expect(await reopened.snapshot(CounterDoc, context)).toEqual({ count: 2 }); + await reopened.close(context); + }); + + it("keeps what host listeners get: exactly what was stored, held changes included", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(NoteDoc, conversationId); + }, context); + const observed: ObservedCommit[] = []; + const stored: CommitPublication[] = []; + session.observeCommits((publication) => { + storage.order.push("observed"); + observed.push(publication); + }); + session.subscribeCommits((publication) => { + storage.order.push("host"); + stored.push(publication); + }); + storage.order.length = 0; + await session.commit( + async (tx) => { + (await tx.doc(NoteDoc, conversationId)).a = 1; + }, + context, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + // Held in memory: observed at once, nothing stored. + expect(storage.order).toEqual(["observed"]); + await session.commit(async (tx) => { + (await tx.doc(NoteDoc, conversationId)).b = 2; + }, context); + expect(storage.order).toEqual(["observed", "adopt", "observed", "adopted", "host", "resolve"]); + expect(documentChanges(observed[1]!).map((change) => change.ops)).toEqual([[["s", ["b"], 2]]]); + expect(stored).toHaveLength(1); + expect(documentChanges(stored[0]!).map((change) => change.ops)).toEqual([ + [ + ["s", ["a"], 1], + ["s", ["b"], 2], + ], + ]); + expect(stored[0]!.seq).toBe((observed[1] as Extract).seq); + await session.close(context); + }); + + it("publishes a store of held changes to host listeners one microtask after its adopt", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(NoteDoc, conversationId); + }, context); + session.subscribeCommits(() => { + storage.order.push(`host adopting=${storage.adopting}`); + }); + await session.commit( + async (tx) => { + (await tx.doc(NoteDoc, conversationId)).a = 1; + }, + context, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + storage.order.length = 0; + await session.unloadDocuments(); + expect(storage.order).toEqual(["adopt", "adopted", "host adopting=false", "resolve"]); + await session.close(context); + }); + + it("publishes a JSONL commit before the storage reclaims its sidecar and resolves", async () => { + /** Node environment that records each sidecar rename, the last step of a reclaim. */ + class RenameLog extends NodeExecutionEnv { + readonly order: string[] = []; + + override renameFile( + source: string, + destination: string, + renameContext: Context, + ): Promise> { + this.order.push(`rename ${basename(destination)}`); + return super.renameFile(source, destination, renameContext); + } + } + // A base at the second count makes JSONL rewrite the document's sidecar after adopting the commit. + const Rebased = defineDoc<{ count: number }>({ + checkpointWhen: (value) => value.count === 2, + kind: "adoption.rebased", + version: 1, + scope: "session", + initial: () => ({ count: 0 }), + }); + const directory = await mkdtemp(join(tmpdir(), "pi-durable-adoption-")); + const env = new RenameLog({ cwd: directory }); + const session = new SessionImpl(await JsonlStorage.open(directory, env, context)); + const count = () => + session.commit(async (tx) => { + (await tx.doc(Rebased)).count++; + }, context); + try { + await count(); + session.subscribeCommits(() => { + env.order.push("host"); + }); + await count(); + env.order.push("resolved"); + expect(env.order).toEqual(["host", expect.stringMatching(/^rename doc-\d+\.jsonl$/), "resolved"]); + } finally { + await session.close(context); + await rm(directory, { recursive: true, force: true }); + } + }); + + it("publishes a store of held changes that Storage rejects after adopting it, then fails the Session", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(NoteDoc, conversationId); + }, context); + const stored: (readonly unknown[])[] = []; + session.subscribeCommits((publication) => { + stored.push(documentChanges(publication).map((change) => change.ops)); + }); + await session.commit( + async (tx) => { + (await tx.doc(NoteDoc, conversationId)).a = 1; + }, + context, + { persistence: { kind: "deferred", withinMs: Infinity } }, + ); + storage.order.length = 0; + storage.fault = "reject_after_adopt"; + await expect(session.unloadDocuments()).rejects.toThrow("reclaim failed"); + expect(storage.order).toEqual(["adopt", "adopted", "resolve"]); + expect(stored).toEqual([[[["s", ["a"], 1]]]]); + expect(await session.closed).toMatchObject({ reason: "failed", error: new Error("reclaim failed") }); + }); +}); diff --git a/packages/durable/test/session-checkpoints-migrations.test.ts b/packages/durable/test/session-checkpoints-migrations.test.ts index 98bb062ab..f5433f1a5 100644 --- a/packages/durable/test/session-checkpoints-migrations.test.ts +++ b/packages/durable/test/session-checkpoints-migrations.test.ts @@ -402,7 +402,7 @@ describe("Session document migrations", () => { // An observer of the older shape. const watch = (await session.watchDoc(Old, context))!; const frames: unknown[] = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { frames.push({ value, ops }); }); expect(await session.snapshot(Current, context)).toEqual({ count: 4 }); @@ -410,7 +410,7 @@ describe("Session document migrations", () => { // An observer of the new shape: the migration changes nothing it sees. const current = (await session.watchDoc(Current, context))!; const currentFrames: unknown[] = []; - current.start(async (value) => { + current.start(async ({ value }) => { currentFrames.push(value); }); @@ -836,7 +836,7 @@ describe("Session tracker cache across definition versions", () => { const watch = (await session.watchDoc(V1Doc, context))!; const frames: (readonly unknown[])[] = []; const delivered = new Promise((resolve) => { - watch.start(async (_value, ops) => { + watch.start(async ({ ops }) => { frames.push(ops); resolve(); }); diff --git a/packages/durable/test/session-deferred-commits.test.ts b/packages/durable/test/session-deferred-commits.test.ts index e4281c5e8..bbf72d002 100644 --- a/packages/durable/test/session-deferred-commits.test.ts +++ b/packages/durable/test/session-deferred-commits.test.ts @@ -562,8 +562,8 @@ describe("publications", () => { const { session, conversationId, observed, fire } = await open(); const watch = (await session.watchDoc(TextDoc, conversationId, context))!; const values: string[] = []; - watch.start(async (value) => { - values.push(value?.text ?? ""); + watch.start(async ({ value }) => { + values.push(value.text); }); const from = observed.length; for (const text of ["a", "b", "c"]) { diff --git a/packages/durable/test/session-document-retention.test.ts b/packages/durable/test/session-document-retention.test.ts index 6e2eed770..c2cec11c8 100644 --- a/packages/durable/test/session-document-retention.test.ts +++ b/packages/durable/test/session-document-retention.test.ts @@ -155,14 +155,14 @@ describe("document cache retention", () => { const kept = (await session.watchDoc(CounterDoc, conversationId, context))!; kept.start(async () => {}); const seen: number[] = []; - watch.start(async (value) => { - seen.push(value?.count ?? -1); + watch.start(async ({ value }) => { + seen.push(value.count); }); clock.now += 1_000_000; await bump(session, conversationId); await settleLine(session); // Still cached: a read long after its retention passed loads nothing from Storage. - await eventually(() => seen.includes(2) && kept.value?.count === 2); + await eventually(() => seen.includes(2) && kept.value.count === 2); const reads = storage.documentReadCount; clock.now += 1_000_000; await session.commit(async (tx) => { @@ -175,7 +175,7 @@ describe("document cache retention", () => { clock.now += 1_000_000; await bump(session, conversationId); await settleLine(session); - await eventually(() => kept.value?.count === 3); + await eventually(() => kept.value.count === 3); await kept.stop(); clock.now += 1_000_000; await session.commit(async () => {}, context); @@ -350,22 +350,27 @@ describe("document cache retention", () => { await session.close(context); }); - it("does not keep a replacement cached for an observer of the retired incarnation", async () => { + it("pins only what is at the address a watch follows: nothing while retired, then the new incarnation", async () => { const { session, sizes } = open(0); const conversationId = await createConversation(session); await bump(session, conversationId); const watch = (await session.watchDoc(CounterDoc, conversationId, context))!; watch.start(async () => {}); await session.commit((tx) => tx.retireDoc(CounterDoc, conversationId), context); - expect(await watch.closed).toEqual({ reason: "retired" }); - expect(watch.value).toBeNull(); - await bump(session, conversationId, 5); await eventually(async () => { await settleLine(session); return sizes().cached === 0; }); - expect(await session.snapshot(CounterDoc, conversationId, context)).toEqual({ count: 5 }); + expect(sizes().attached).toBe(0); + await bump(session, conversationId, 5); + await settleLine(session); + expect(sizes()).toMatchObject({ cached: 1, attached: 1 }); + await eventually(async () => watch.value.count === 5); await watch.stop(); + await eventually(async () => { + await settleLine(session); + return sizes().cached === 0; + }); await session.close(context); }); @@ -531,7 +536,7 @@ describe("document cache retention", () => { const { harness, root } = await openChat(new MemoryStorage(), setup); const stream = await watchEvents(harness, root.id, context); const events: AgentEvent[] = []; - stream.start(async (batch) => { + stream.start(async ({ value: batch }) => { events.push(...batch); }); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); diff --git a/packages/durable/test/session-observers.test.ts b/packages/durable/test/session-observers.test.ts new file mode 100644 index 000000000..c0d854bb9 --- /dev/null +++ b/packages/durable/test/session-observers.test.ts @@ -0,0 +1,411 @@ +// Tests for `Session.observeDoc()`, the synchronous observer of one document address that watches are built on. +import { setFlagsFromString } from "node:v8"; +import { runInNewContext } from "node:vm"; +import type { WatchChange } from "@earendil-works/chord"; +import { withAbortSignal } from "@earendil-works/chord/context"; +import { + type ConversationId, + defineDoc, + MemoryStorage, + type Observation, + type Observer, + type ReportInfo, +} from "@earendil-works/pi-durable"; +import { describe, expect, it } from "vitest"; +import { SessionImpl } from "../src/session/session.ts"; +import { ControlledStorage, context, createConversation, flush } from "./session-support.ts"; + +type Counter = { count: number }; + +const SessionCounter = defineDoc({ + checkpointWhen: () => false, + kind: "observe.session-counter", + version: 1, + scope: "session", + initial: () => ({ count: 0 }), +}); + +/** A latest, initial-fork conversation document, the kind a deferred commit may hold. */ +const ConversationCounter = defineDoc({ + checkpointWhen: () => false, + kind: "observe.conversation-counter", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ count: 0 }), +}); + +/** A Session that keeps its reports, which a plain Session drops. */ +class ReportingSession extends SessionImpl { + readonly reports: { readonly error: unknown; readonly info: ReportInfo }[] = []; + + override report(error: unknown, info: ReportInfo = { type: "listener_error" }): void { + this.reports.push({ error, info }); + } +} + +/** An observer that records each call, and whether it ran inside the storage's `adopt`. */ +function recorder(storage: ControlledStorage) { + const calls: Array< + | { readonly advance: WatchChange>; readonly adopting: boolean } + | { readonly end: { readonly error: unknown } | undefined } + > = []; + const observer: Observer> = { + advance: (change) => { + calls.push({ advance: change, adopting: storage.adopting }); + }, + end: (failure) => { + calls.push({ end: failure }); + }, + }; + return { calls, observer }; +} + +function open() { + const storage = new ControlledStorage(); + const session = new ReportingSession(storage); + return { storage, session }; +} + +const add = (session: SessionImpl, by = 1) => + session.commit(async (tx) => { + (await tx.doc(SessionCounter)).count += by; + }, context); + +describe("observeDoc", () => { + it("returns undefined for an absent address and attaches nothing", async () => { + const { storage, session } = open(); + const { calls, observer } = recorder(storage); + expect(await session.observeDoc(SessionCounter, observer, context)).toBeUndefined(); + await add(session); + expect(calls).toEqual([]); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); + + it("starts at the committed value and advances inside adopt with each stored commit's ops", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + expect(observation.value).toEqual({ count: 1 }); + const committing = add(session, 2); + await committing; + expect(calls).toEqual([ + { advance: { value: { count: 3 }, ops: [["s", ["count"], 3]], retired: false }, adopting: true }, + ]); + observation.detach(); + await session.close(context); + }); + + it("advances when memory adopts a held commit, before anything is stored", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(ConversationCounter, conversationId); + }, context); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(ConversationCounter, conversationId, observer, context))!; + const stored = storage.commits.length; + await session.commit( + async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 5; + }, + context, + { persistence: { kind: "deferred", withinMs: Number.POSITIVE_INFINITY } }, + ); + expect(storage.commits).toHaveLength(stored); + expect(calls).toEqual([ + { advance: { value: { count: 5 }, ops: [["s", ["count"], 5]], retired: false }, adopting: false }, + ]); + observation.detach(); + await session.close(context); + }); + + it("follows a conversation document's address: a retirement with the last value, then the new incarnation", async () => { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 2; + }, context); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(ConversationCounter, conversationId, observer, context))!; + await session.commit((tx) => tx.retireDoc(ConversationCounter, conversationId), context); + // The address is absent now: a new attachment finds nothing, the existing one goes on. + expect(await session.observeDoc(ConversationCounter, conversationId, recorder(storage).observer, context)).toBe( + undefined, + ); + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 9; + }, context); + expect(calls).toEqual([ + { advance: { value: observation.value, ops: [], retired: true }, adopting: true }, + { advance: { value: { count: 9 }, ops: [["r", { count: 9 }]], retired: false }, adopting: true }, + ]); + expect((calls[0] as { advance: WatchChange }).advance.value).toBe(observation.value); + observation.detach(); + await session.close(context); + }); + + it("gets nothing after detach, which is idempotent and unpins the document", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + expect(session.documentCacheSizes.attached).toBe(1); + observation.detach(); + observation.detach(); + expect(session.documentCacheSizes.attached).toBe(0); + await add(session); + await session.close(context); + expect(calls).toEqual([]); + }); + + it("ends once when the Session closes", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + await session.observeDoc(SessionCounter, observer, context); + await session.close(context); + expect(calls).toEqual([{ end: undefined }]); + }); + + it("ends with the error that failed the Session", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + await session.observeDoc(SessionCounter, observer, context); + const failure = new Error("disk gone"); + storage.failNextCommit(failure); + await expect(add(session)).rejects.toThrow("disk gone"); + await session.closed; + expect(calls).toEqual([{ end: { error: failure } }]); + }); + + it("reports a throwing observer, detaches it and ends it with the error; the commit and other observers go on", async () => { + const { storage, session } = open(); + await add(session); + const bug = new Error("observer bug"); + const ends: unknown[] = []; + const throwing: Observer> = { + advance: () => { + throw bug; + }, + end: (failure) => ends.push(failure), + }; + await session.observeDoc(SessionCounter, throwing, context); + const healthy = recorder(storage); + await session.observeDoc(SessionCounter, healthy.observer, context); + await add(session); + await add(session); + expect(session.reports).toEqual([{ error: bug, info: { type: "listener_error" } }]); + expect(ends).toEqual([{ error: bug }]); + expect(healthy.calls).toHaveLength(2); + expect(await session.snapshot(SessionCounter, context)).toEqual({ count: 3 }); + await session.close(context); + // Detached at its throw: the close does not end it again. + expect(ends).toHaveLength(1); + expect(session.failure).toBeUndefined(); + }); + + it("attaches atomically on the line: a commit queued before the attachment is in its value", async () => { + const { storage, session } = open(); + await add(session); + const { calls, observer } = recorder(storage); + const committing = add(session, 10); + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + await committing; + expect(observation.value).toEqual({ count: 11 }); + expect(calls).toEqual([]); + observation.detach(); + await session.close(context); + }); + + it("rejects an attachment whose context is cancelled, and attaches nothing", async () => { + const { storage, session } = open(); + await add(session); + const controller = new AbortController(); + controller.abort(new Error("gone")); + const cancelled = withAbortSignal(controller.signal, context); + await expect(session.observeDoc(SessionCounter, recorder(storage).observer, cancelled)).rejects.toThrow("gone"); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); + + it("keeps no detached observer alive", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const session = new SessionImpl(new MemoryStorage()); + await add(session); + /** Attach an observer only the Session references; `detach` says whether to detach it before returning. */ + const attach = async (detach: boolean) => { + const observer: Observer> = { advance: () => {}, end: () => {} }; + const observation = (await session.observeDoc(SessionCounter, observer, context))!; + if (detach) observation.detach(); + return { observation, observer: new WeakRef(observer) }; + }; + // Control: an attached observer stays, so the collection below is real. + const attached = await attach(false); + await collect(); + expect(attached.observer.deref()).toBeDefined(); + attached.observation.detach(); + + const detached = await attach(true); + await collect(); + expect(detached.observer.deref()).toBeUndefined(); + await flush(); + await session.close(context); + }); + + it("rejects, never throws, on a closed Session or a missing scope ID", async () => { + const { storage, session } = open(); + const missing = session.observeDoc( + ConversationCounter, + undefined as unknown as ConversationId, + recorder(storage).observer, + context, + ); + await expect(missing).rejects.toThrow(TypeError); + await session.close(context); + const closed = session.observeDoc(SessionCounter, recorder(storage).observer, context); + await expect(closed).rejects.toThrow("Session is closed"); + }); + it("keeps no value of a document retired for good, while still attached", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const session = new SessionImpl(new MemoryStorage()); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + await tx.doc(ConversationCounter, conversationId); + }, context); + /** Attach an observer that keeps nothing, change the document, and retire it: only a WeakRef to its last value. */ + const retire = async () => { + let last: WeakRef | undefined; + const observation = (await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: (change) => { + if (!change.retired) last = new WeakRef(change.value); + }, + end: () => {}, + }, + context, + ))!; + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 7; + }, context); + await session.commit((tx) => tx.retireDoc(ConversationCounter, conversationId), context); + await session.unloadDocuments(); + return { observation, last: last! }; + }; + const { observation, last } = await retire(); + await collect(); + expect(last.deref()).toBeUndefined(); + observation.detach(); + await session.close(context); + }); +}); + +describe("observeDoc detached during a commit", () => { + /** A Session with a conversation counter at 1, and the commit that retires it and creates it again at 2. */ + async function retiring() { + const { storage, session } = open(); + const conversationId = await createConversation(session); + await session.commit(async (tx) => { + (await tx.doc(ConversationCounter, conversationId)).count = 1; + }, context); + const recreate = () => + session.commit(async (tx) => { + await tx.retireDoc(ConversationCounter, conversationId); + (await tx.doc(ConversationCounter, conversationId)).count = 2; + }, context); + return { storage, session, conversationId, recreate }; + } + + it("gets nothing more of the commit once it throws on the retirement, and pins nothing for the recreation", async () => { + const { session, conversationId, recreate } = await retiring(); + const bug = new Error("observer bug"); + const changes: WatchChange>[] = []; + const ends: unknown[] = []; + const observation = (await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: (change) => { + changes.push(change); + throw bug; + }, + end: (failure) => ends.push(failure), + }, + context, + ))!; + await recreate(); + expect(changes).toEqual([{ value: observation.value, ops: [], retired: true }]); + expect(ends).toEqual([{ error: bug }]); + expect(session.reports).toEqual([{ error: bug, info: { type: "listener_error" } }]); + expect(session.documentCacheSizes.attached).toBe(0); + observation.detach(); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); + + it("gets nothing more of the commit once it detaches in the retirement call, and pins nothing", async () => { + const { session, conversationId, recreate } = await retiring(); + const changes: WatchChange>[] = []; + const ends: unknown[] = []; + let observation: Observation> | undefined; + observation = await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: (change) => { + changes.push(change); + observation?.detach(); + }, + end: (failure) => ends.push(failure), + }, + context, + ); + await recreate(); + expect(changes).toEqual([{ value: observation!.value, ops: [], retired: true }]); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + expect(ends).toEqual([]); + }); + + it("gets nothing of the commit when another observer's call detached it first, and pins nothing", async () => { + const { storage, session, conversationId, recreate } = await retiring(); + let second: Observation> | undefined; + const first = (await session.observeDoc( + ConversationCounter, + conversationId, + { + advance: () => second?.detach(), + end: () => {}, + }, + context, + ))!; + const detached = recorder(storage); + second = await session.observeDoc(ConversationCounter, conversationId, detached.observer, context); + await recreate(); + expect(detached.calls).toEqual([]); + // Only the first observer pins the new incarnation; its detach leaves nothing pinned. + first.detach(); + expect(session.documentCacheSizes.attached).toBe(0); + await session.close(context); + }); +}); diff --git a/packages/durable/test/session-support.ts b/packages/durable/test/session-support.ts index e3bd17d44..98e3a54ce 100644 --- a/packages/durable/test/session-support.ts +++ b/packages/durable/test/session-support.ts @@ -43,6 +43,10 @@ export class ControlledStorage extends MemoryStorage { readonly commits: (readonly StorageWrite[])[] = []; mintCount = 0; documentReadCount = 0; + /** Set while `adopt` runs, the storage's synchronous step: code that sees it set runs inside `Storage.commit()`'s adopt. */ + adopting = false; + /** Runs once, in the next commit's synchronous step right after `adopt` returned. */ + afterAdopt: (() => void) | undefined; #commitGate: { gate: Deferred; entered: Deferred } | undefined; #findGate: { gate: Deferred; entered: Deferred } | undefined; #commitFailure: Error | undefined; @@ -97,7 +101,18 @@ export class ControlledStorage extends MemoryStorage { this.#commitFailure = undefined; throw failure; } - return super.commit(writes, adopt, commitContext); + const step = (seq: Seq): void => { + this.adopting = true; + try { + adopt(seq); + } finally { + this.adopting = false; + } + const after = this.afterAdopt; + this.afterAdopt = undefined; + after?.(); + }; + return super.commit(writes, step, commitContext); } override mintId>(): Promise { @@ -124,6 +139,39 @@ export class ControlledStorage extends MemoryStorage { } } +/** + * Controlled storage whose commits resolve a macrotask after they adopted the batch, as a storage with work after + * durability does (JSONL reclaims sidecars): code woken inside `adopt` runs before the commit resolves. `gap` runs in + * that work, a macrotask after adoption, and `rejectAfterAdopt` makes a commit it matches reject there, durable and + * adopted, as a failed reclaim does; each fires once. + */ +export class LateResolvingStorage extends ControlledStorage { + /** Set from a commit's start until it resolves. */ + committing = false; + gap: ((writes: readonly StorageWrite[]) => boolean) | undefined; + rejectAfterAdopt: ((writes: readonly StorageWrite[]) => boolean) | undefined; + + override async commit(writes: readonly StorageWrite[], adopt: (seq: Seq) => void, commitContext: Context) { + this.committing = true; + try { + const seq = await super.commit(writes, adopt, commitContext); + await flush(); + if (this.gap?.(writes) === true) { + this.gap = undefined; + // What the gap started runs before the commit resolves. + await flush(); + } + if (this.rejectAfterAdopt?.(writes) === true) { + this.rejectAfterAdopt = undefined; + throw new Error("reclaim failed"); + } + return seq; + } finally { + this.committing = false; + } + } +} + /** Session kernel plus its controlled storage and every committed publication. */ export function openTestSession(): { readonly storage: ControlledStorage; diff --git a/packages/durable/test/session-watches.test.ts b/packages/durable/test/session-watches.test.ts index 79a7f5b39..cb06268a7 100644 --- a/packages/durable/test/session-watches.test.ts +++ b/packages/durable/test/session-watches.test.ts @@ -1,6 +1,6 @@ import { setFlagsFromString } from "node:v8"; import { runInNewContext } from "node:vm"; -import type { Context } from "@earendil-works/chord"; +import type { Context, WatchChange, WatchEnd } from "@earendil-works/chord"; import { BACKGROUND_CONTEXT, createContextKey, @@ -9,10 +9,10 @@ import { withContextValue, } from "@earendil-works/chord/context"; import { applyImmutable } from "@earendil-works/chord/delta"; -import { type DocumentWatch, defineDoc, MemoryStorage } from "@earendil-works/pi-durable"; +import { type DocumentWatch, defineDoc, defineDocFamily, MemoryStorage } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { SessionImpl } from "../src/session/session.ts"; -import { context, documentChanges, flush, openTestSession } from "./session-support.ts"; +import { context, createConversation, documentChanges, flush, openTestSession } from "./session-support.ts"; type State = { value: number; items: string[]; retained: { label: string } }; @@ -44,6 +44,15 @@ function deferred(): Deferred { return { promise, resolve }; } +/** The end of `closed`, or undefined while the watch is open. */ +function endOf(watch: { readonly closed: Promise }): { readonly end: WatchEnd | undefined } { + const settled: { end: WatchEnd | undefined } = { end: undefined }; + void watch.closed.then((end) => { + settled.end = end; + }); + return settled; +} + async function createState(): Promise> { const harness = openTestSession(); await harness.session.commit((tx) => tx.doc(StateDoc).then(() => undefined), context); @@ -76,8 +85,8 @@ describe("Session document watches", () => { expect(watch.value).toBe(initial); let inline = true; - const deliveries: Array<{ value: Readonly | null; ops: unknown }> = []; - watch.start(async (value, ops) => { + const deliveries: Array<{ value: Readonly; ops: unknown }> = []; + watch.start(async ({ value, ops }) => { expect(inline).toBe(false); expect(watch.value).toBe(value); deliveries.push({ value, ops }); @@ -112,10 +121,10 @@ describe("Session document watches", () => { const batches: unknown[] = []; let active = 0; let maxActive = 0; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { active++; maxActive = Math.max(maxActive, active); - values.push(value?.value ?? -1); + values.push(value.value); batches.push(ops); if (values.length === 1) { entered.resolve(); @@ -146,8 +155,8 @@ describe("Session document watches", () => { const watch = (await session.watchDoc(StateDoc, context))!; const completed = deferred(); const values: number[] = []; - watch.start(async (value) => { - const current = value?.value ?? -1; + watch.start(async ({ value }) => { + const current = value.value; values.push(current); if (current === 1) { await session.commit(async (tx) => { @@ -174,8 +183,8 @@ describe("Session document watches", () => { }, context); } const deliveries: Array<{ value: number; ops: unknown }> = []; - watch.start(async (value, ops) => { - deliveries.push({ value: value?.value ?? -1, ops }); + watch.start(async ({ value, ops }) => { + deliveries.push({ value: value.value, ops }); }); await flush(); expect(deliveries).toEqual([{ value: 101, ops: [["s", ["value"], 101]] }]); @@ -197,13 +206,13 @@ describe("Session document watches", () => { for (let index = from; index < to; index++) state.keys[`k${index}`] = index; }, context); } - const deliveries: Array<{ value: Readonly | null; ops: unknown }> = []; - watch.start(async (value, ops) => { + const deliveries: Array<{ value: Readonly; ops: unknown }> = []; + watch.start(async ({ value, ops }) => { deliveries.push({ value, ops }); }); await flush(); expect(deliveries).toEqual([{ value: watch.value, ops: [["r", watch.value]] }]); - expect(Object.keys(watch.value?.keys ?? {})).toHaveLength(10_001); + expect(Object.keys(watch.value.keys)).toHaveLength(10_001); await watch.stop(); }); @@ -213,8 +222,8 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); const deliveries: Array<{ value: number; ops: unknown }> = []; - watch.start(async (value, ops) => { - deliveries.push({ value: value?.value ?? -1, ops }); + watch.start(async ({ value, ops }) => { + deliveries.push({ value: value.value, ops }); if (deliveries.length === 1) { entered.resolve(); await release.promise; @@ -237,7 +246,7 @@ describe("Session document watches", () => { await watch.stop(); }); - it("folds retirement over pending commits and then closes", async () => { + it("folds a retirement into the pending commits' call, which reports it with the last value, and goes on", async () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; for (let value = 1; value <= 100; value++) { @@ -245,13 +254,22 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = value; }, context); } + const last = (await session.snapshot(StateDoc, context))!; await session.commit((tx) => tx.retireDoc(StateDoc), context); - const deliveries: Array<{ value: Readonly | null; ops: unknown }> = []; - watch.start(async (value, ops) => { - deliveries.push({ value, ops }); + const changes: WatchChange>[] = []; + watch.start(async (change) => { + changes.push(change); }); - expect(await watch.closed).toEqual({ reason: "retired" }); - expect(deliveries).toEqual([{ value: null, ops: [["r", null]] }]); + await flush(); + expect(changes).toEqual([{ value: last, ops: [["s", ["value"], 100]], retired: true }]); + expect(changes[0]!.value).toBe(last); + expect(watch.value).toBe(last); + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).value = 7; + }, context); + await flush(); + expect(changes[1]).toEqual({ value: { ...last, value: 7 }, ops: [["r", { ...last, value: 7 }]], retired: false }); + await watch.stop(); }); it("delivers replayable structural no-op commits instead of suppressing them", async () => { @@ -267,7 +285,7 @@ describe("Session document watches", () => { expect(newest).not.toBe(initial); expect(newest).toEqual(initial); const batches: unknown[] = []; - watch.start(async (value, ops) => { + watch.start(async ({ value, ops }) => { expect(value).toBe(newest); batches.push(ops); }); @@ -290,7 +308,7 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); let deliveryContext: Context | undefined; - watch.start(async (_value, _ops, delivered) => { + watch.start(async (_change, delivered) => { deliveryContext = delivered; entered.resolve(); await release.promise; @@ -312,8 +330,8 @@ describe("Session document watches", () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; const initial = watch.value; - let delivered: Readonly | null | undefined; - watch.start(async (value) => { + let delivered: Readonly | undefined; + watch.start(async ({ value }) => { delivered = value; }); await session.commit(async (tx) => { @@ -322,34 +340,55 @@ describe("Session document watches", () => { await flush(); expect(delivered).toBe(watch.value); expect(delivered).not.toBe(initial); - expect(delivered?.retained).toBe(initial?.retained); - expect(initial?.value).toBe(0); + expect(delivered?.retained).toBe(initial.retained); + expect(initial.value).toBe(0); await watch.stop(); }); - it("delivers retirement and does not follow recreation", async () => { + it("reports a change and a retirement in one commit as the retirement, with the value it was given before", async () => { + const { session, storage } = await createState(); + const watch = (await session.watchDoc(StateDoc, context))!; + const before = watch.value; + const changes: WatchChange>[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).value = 5; + await tx.retireDoc(StateDoc); + }, context); + await flush(); + // Storage got the change, then the retirement; the commit publishes the retirement only. + expect(storage.commits.at(-1)!.map((write) => write.type)).toEqual(["document.change", "document.retire"]); + expect(changes).toEqual([{ value: before, ops: [], retired: true }]); + expect(changes[0]!.value).toBe(before); + await watch.stop(); + }); + + it("follows its address across a retirement and a recreation in one commit, in one call", async () => { const { session } = await createState(); - const oldWatch = (await session.watchDoc(StateDoc, context))!; - const values: Array | null> = []; - oldWatch.start(async (value) => { - values.push(value); + const watch = (await session.watchDoc(StateDoc, context))!; + const settled = endOf(watch); + const changes: WatchChange>[] = []; + watch.start(async (change) => { + changes.push(change); }); await session.commit(async (tx) => { await tx.retireDoc(StateDoc); (await tx.doc(StateDoc)).value = 10; }, context); - expect(await oldWatch.closed).toEqual({ reason: "retired" }); - expect(values).toEqual([null]); - expect(oldWatch.value).toBeNull(); - - const replacement = (await session.watchDoc(StateDoc, context))!; - expect(replacement.value?.value).toBe(10); + await flush(); + const recreated = (await session.snapshot(StateDoc, context))!; + expect(changes).toHaveLength(1); + expect(changes[0]).toEqual({ value: recreated, ops: [["r", recreated]], retired: true }); await session.commit(async (tx) => { (await tx.doc(StateDoc)).value = 11; }, context); await flush(); - expect(oldWatch.value).toBeNull(); - await replacement.stop(); + expect(changes[1]).toEqual({ value: watch.value, ops: [["s", ["value"], 11]], retired: false }); + expect(watch.value.value).toBe(11); + expect(settled.end).toBeUndefined(); + await watch.stop(); }); it("Session close discards retirement buffered before start", async () => { @@ -368,9 +407,9 @@ describe("Session document watches", () => { const watch = (await session.watchDoc(StateDoc, context))!; const entered = deferred(); const release = deferred(); - const values: Array = []; - watch.start(async (value) => { - values.push(value?.value ?? null); + const values: number[] = []; + watch.start(async ({ value }) => { + values.push(value.value); entered.resolve(); await release.promise; }); @@ -421,7 +460,7 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); let deliveryContext: Context | undefined; - watch.start(async (_value, _ops, delivered) => { + watch.start(async (_change, delivered) => { deliveryContext = delivered; entered.resolve(); await release.promise; @@ -442,7 +481,7 @@ describe("Session document watches", () => { const entered = deferred(); const release = deferred(); let deliveryContext: Context | undefined; - watch.start(async (_value, _ops, delivered) => { + watch.start(async (_change, delivered) => { deliveryContext = delivered; entered.resolve(); await release.promise; @@ -503,8 +542,8 @@ describe("Session document watches", () => { expect(watch.value).toEqual({ value: 3, migrated: true }); expect(storage.commits).toHaveLength(commits); const values: number[] = []; - watch.start(async (value) => { - values.push(value?.value ?? -1); + watch.start(async ({ value }) => { + values.push(value.value); }); await session.commit((tx) => tx.doc(Current).then(() => undefined), context); await flush(); @@ -520,8 +559,8 @@ describe("Session document watches", () => { it("can replay every delivered exact operation batch from the acquisition revision", async () => { const { session } = await createState(); const watch = (await session.watchDoc(StateDoc, context))!; - let replica: Readonly | null = watch.value; - watch.start(async (value, ops) => { + let replica: Readonly = watch.value; + watch.start(async ({ value, ops }) => { replica = applyImmutable(replica, ops); expect(replica).toEqual(value); }); @@ -550,10 +589,195 @@ describe("Session document watches", () => { (await tx.doc(StateDoc)).value = 7; }, context); await flush(); - expect(watch.value?.value).toBe(7); + expect(watch.value.value).toBe(7); await watch.stop(); }); + it("ignores the same kind in another conversation and under another family key", async () => { + const Notes = defineDoc<{ text: string }>({ + checkpointWhen: () => false, + kind: "watch.scoped-notes", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ text: "" }), + }); + const Members = defineDocFamily<{ text: string }, null>({ + checkpointWhen: () => false, + kind: "watch.members", + version: 1, + family: true, + scope: "session", + initial: () => ({ text: "" }), + }); + const { session } = openTestSession(); + const watched = await createConversation(session); + const other = await createConversation(session); + await session.commit(async (tx) => { + (await tx.doc(Notes, watched)).text = "a"; + (await tx.doc(Members, "watched", null)).text = "a"; + }, context); + const notes = (await session.watchDoc(Notes, watched, context))!; + const member = (await session.watchDoc(Members, "watched", context))!; + const changes: WatchChange>[] = []; + notes.start(async (change) => { + changes.push(change); + }); + member.start(async (change) => { + changes.push(change); + }); + // The same kinds at other addresses: created, changed, retired, and created again. + await session.commit(async (tx) => { + (await tx.doc(Notes, other)).text = "other"; + (await tx.doc(Members, "other", null)).text = "other"; + }, context); + await session.commit(async (tx) => { + await tx.retireDoc(Notes, other); + await tx.retireDoc(Members, "other"); + }, context); + await session.commit(async (tx) => { + (await tx.doc(Notes, other)).text = "again"; + (await tx.doc(Members, "other", null)).text = "again"; + }, context); + await flush(); + expect(changes).toEqual([]); + expect(session.documentCacheSizes.attached).toBe(2); + await notes.stop(); + await member.stop(); + await session.close(context); + }); + + it("takes the version of a recreation written by a newer definition, then its ops", async () => { + const Old = defineDoc<{ value: number }>({ + checkpointWhen: () => false, + kind: "watch.recreated-version", + version: 1, + scope: "session", + initial: () => ({ value: 1 }), + }); + const Current = defineDoc<{ value: number; migrated: boolean }>({ + checkpointWhen: () => false, + kind: "watch.recreated-version", + version: 2, + scope: "session", + initial: () => ({ value: 0, migrated: false }), + migrate: (value) => ({ value: value.value as number, migrated: true }), + }); + const { session } = openTestSession(); + await session.commit((tx) => tx.doc(Old).then(() => undefined), context); + const watch = (await session.watchDoc(Old, context))!; + const changes: WatchChange[] = []; + watch.start(async (change) => { + changes.push(change); + }); + await session.commit(async (tx) => { + await tx.retireDoc(Current); + (await tx.doc(Current)).value = 5; + }, context); + await flush(); + await session.commit(async (tx) => { + (await tx.doc(Current)).value = 6; + }, context); + await flush(); + expect(changes).toEqual([ + { + value: { value: 5, migrated: false }, + ops: [["r", { value: 5, migrated: false }]], + retired: true, + }, + { value: { value: 6, migrated: false }, ops: [["s", ["value"], 6]], retired: false }, + ]); + await watch.stop(); + await session.close(context); + }); + + it("keeps no earlier incarnation alive after a retirement and a recreation, while it goes on", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + /** Watch the document, retire and recreate it, and deliver the new value; `drop` then unloads the cache. */ + const follow = async (session: SessionImpl, drop: boolean) => { + await session.commit(async (tx) => { + (await tx.doc(StateDoc)).retained = { label: "first incarnation" }; + }, context); + const watch = (await session.watchDoc(StateDoc, context))!; + const first = new WeakRef(watch.value.retained); + watch.start(async () => {}); + await session.commit(async (tx) => { + await tx.retireDoc(StateDoc); + (await tx.doc(StateDoc)).value = 2; + }, context); + await flush(); + if (drop) await session.unloadDocuments(); + return { watch, first }; + }; + // Retention 0 evicts the retired incarnation at once; the default keeps the cache until it is unloaded. + for (const [session, drop] of [ + [new SessionImpl(new MemoryStorage(), Date.now, () => 0), false], + [new SessionImpl(new MemoryStorage(), Date.now, () => 600_000), true], + ] as const) { + const { watch, first } = await follow(session, drop); + await collect(); + expect(watch.value.value).toBe(2); + expect(watch.value.retained).toEqual({ label: "stable" }); + expect(first.deref()).toBeUndefined(); + await watch.stop(); + await session.close(context); + } + }); + + it("keeps a watch of a retired conversation document until it is stopped, then nothing of it", async () => { + setFlagsFromString("--expose-gc"); + const gc = runInNewContext("gc") as () => void; + const collect = async (): Promise => { + for (let round = 0; round < 5; round++) { + await new Promise((resolve) => setTimeout(resolve, 0)); + gc(); + } + }; + const Notes = defineDoc<{ text: string }>({ + checkpointWhen: () => false, + kind: "watch.notes", + version: 1, + scope: "conversation", + history: "latest", + fork: "initial", + initial: () => ({ text: "" }), + }); + const { session } = openTestSession(); + const conversationId = await createConversation(session); + /** A started watch of a document that is then retired; `stop` says whether to stop it before returning. */ + const acquire = async (stop: boolean) => { + await session.commit(async (tx) => { + (await tx.doc(Notes, conversationId)).text = "kept"; + }, context); + const watch = (await session.watchDoc(Notes, conversationId, context))!; + const listener = async (): Promise => {}; + watch.start(listener); + await session.commit((tx) => tx.retireDoc(Notes, conversationId), context); + await flush(); + if (stop) await watch.stop(); + return { watch: new WeakRef(watch), listener: new WeakRef(listener) }; + }; + // A watch follows its address, so a forgotten one stays until it is stopped or the Session ends. + const forgotten = await acquire(false); + await collect(); + expect(forgotten.watch.deref()).toBeDefined(); + await forgotten.watch.deref()!.stop(); + + const stopped = await acquire(true); + await collect(); + expect(stopped.watch.deref()).toBeUndefined(); + expect(stopped.listener.deref()).toBeUndefined(); + await session.close(context); + }); + it("retains neither its Session nor its listener once stopped, while still referenced", async () => { setFlagsFromString("--expose-gc"); const gc = runInNewContext("gc") as () => void; @@ -595,7 +819,7 @@ describe("Session document watches", () => { expect(stopped.session.deref()).toBeUndefined(); expect(stopped.listener.deref()).toBeUndefined(); // What it delivered last stays readable. - expect(stopped.watch.value?.value).toBe(1); + expect(stopped.watch.value.value).toBe(1); expect(await stopped.watch.closed).toEqual({ reason: "stopped" }); }); }); diff --git a/packages/durable/test/task-support.ts b/packages/durable/test/task-support.ts index a94373050..77cac5897 100644 --- a/packages/durable/test/task-support.ts +++ b/packages/durable/test/task-support.ts @@ -71,6 +71,7 @@ export async function openTasks( readonly settings?: HarnessSettings; readonly onWake?: HarnessOptions["onWake"]; readonly setTimer?: HarnessOptions["setTimer"]; + readonly telemetry?: HarnessOptions["telemetry"]; } = {}, ): Promise<{ readonly harness: Harness; @@ -95,6 +96,7 @@ export async function openTasks( ...(options.settings === undefined ? {} : { settings: options.settings }), ...(options.onWake === undefined ? {} : { onWake: options.onWake }), ...(options.setTimer === undefined ? {} : { setTimer: options.setTimer }), + ...(options.telemetry === undefined ? {} : { telemetry: options.telemetry }), }, context, ); diff --git a/packages/env/src/remote-env.ts b/packages/env/src/remote-env.ts index c77ce201e..f3740994b 100644 --- a/packages/env/src/remote-env.ts +++ b/packages/env/src/remote-env.ts @@ -9,6 +9,7 @@ import { type ExecutionEnv, ExecutionError, err, + type FileChange, FileError, type FileInfo, type FileKind, @@ -25,7 +26,6 @@ import { StreamDecoder, type TextLine, type TextLineReader, - type WatchChange, type WatchTarget, } from "@earendil-works/pi-durable/env"; import { type Connection, type Json, nodePlatform, RemoteError, type RemoteInfo } from "./connection.ts"; @@ -826,7 +826,7 @@ export class RemoteExecutionEnv implements ExecutionEnv { async watch( targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, context: Context, ): Promise> { const aborted = abortResult(context.abortSignal); diff --git a/packages/env/src/watch.ts b/packages/env/src/watch.ts index 402869891..3ee9c0fd8 100644 --- a/packages/env/src/watch.ts +++ b/packages/env/src/watch.ts @@ -1,5 +1,5 @@ import type { Context } from "@earendil-works/chord"; -import { FileError, type FileWatcher, type WatchChange, type WatchTarget } from "@earendil-works/pi-durable/env"; +import { type FileChange, FileError, type FileWatcher, type WatchTarget } from "@earendil-works/pi-durable/env"; import { type Connection, isConnectionLost } from "./connection.ts"; import { toFileError } from "./errors.ts"; @@ -23,7 +23,7 @@ const RECONNECT_MAX_MS = 30_000; export class RemoteWatcher implements FileWatcher { readonly #connection: Connection; readonly #targets: readonly WatchTarget[]; - readonly #onChange: (change: WatchChange) => void; + readonly #onChange: (change: FileChange) => void; readonly #options: RemoteWatchOptions; #mode: "native" | "polling" = "native"; #closed = false; @@ -34,7 +34,7 @@ export class RemoteWatcher implements FileWatcher { private constructor( connection: Connection, targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, options: RemoteWatchOptions, ) { this.#connection = connection; @@ -47,7 +47,7 @@ export class RemoteWatcher implements FileWatcher { static async open( connection: Connection, targets: readonly WatchTarget[], - onChange: (change: WatchChange) => void, + onChange: (change: FileChange) => void, options: RemoteWatchOptions = {}, ): Promise { const watcher = new RemoteWatcher(connection, targets, onChange, options); @@ -67,7 +67,7 @@ export class RemoteWatcher implements FileWatcher { await this.#request; } - #deliver(change: WatchChange): void { + #deliver(change: FileChange): void { if (this.#closed) return; try { this.#onChange(change);